1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the ARM target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_mod_imm(int64_t Imm) {
12
13 return ARM_AM::getSOImmVal(Imm) != -1;
14
15}
16static bool Predicate_imm0_65535(int64_t Imm) {
17
18 return Imm >= 0 && Imm < 65536;
19
20}
21static bool Predicate_imm0_7(int64_t Imm) {
22
23 return Imm >= 0 && Imm < 8;
24
25}
26static bool Predicate_imm8_255(int64_t Imm) {
27
28 return Imm >= 8 && Imm < 256;
29
30}
31static bool Predicate_imm0_255(int64_t Imm) {
32 return Imm >= 0 && Imm < 256;
33}
34static bool Predicate_t2_so_imm(int64_t Imm) {
35
36 return ARM_AM::getT2SOImmVal(Imm) != -1;
37
38}
39static bool Predicate_imm0_4095(int64_t Imm) {
40
41 return Imm >= 0 && Imm < 4096;
42
43}
44static bool Predicate_imm1_31(int64_t Imm) {
45 return Imm > 0 && Imm < 32;
46}
47static bool Predicate_imm0_31(int64_t Imm) {
48
49 return Imm >= 0 && Imm < 32;
50
51}
52static bool Predicate_shr_imm8(int64_t Imm) {
53 return Imm > 0 && Imm <= 8;
54}
55static bool Predicate_shr_imm16(int64_t Imm) {
56 return Imm > 0 && Imm <= 16;
57}
58static bool Predicate_shr_imm32(int64_t Imm) {
59 return Imm > 0 && Imm <= 32;
60}
61static bool Predicate_VectorIndex32(int64_t Imm) {
62
63 return ((uint64_t)Imm) < 2;
64
65}
66static bool Predicate_t2_so_imm_neg(int64_t Imm) {
67
68 return Imm && ARM_AM::getT2SOImmVal(-(uint32_t)Imm) != -1;
69
70}
71static bool Predicate_imm0_15(int64_t Imm) {
72
73 return Imm >= 0 && Imm < 16;
74
75}
76
77
78// FastEmit functions for ARMISD::CALL.
79
80unsigned fastEmit_ARMISD_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
81 if (RetVT.SimpleTy != MVT::isVoid)
82 return 0;
83 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
84 return fastEmitInst_r(ARM::BLX_noip, &ARM::GPRnoipRegClass, Op0, Op0IsKill);
85 }
86 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( !MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
87 return fastEmitInst_r(ARM::BLX, &ARM::GPRRegClass, Op0, Op0IsKill);
88 }
89 return 0;
90}
91
92unsigned fastEmit_ARMISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
93 switch (VT.SimpleTy) {
94 case MVT::i32: return fastEmit_ARMISD_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill);
95 default: return 0;
96 }
97}
98
99// FastEmit functions for ARMISD::CALL_NOLINK.
100
101unsigned fastEmit_ARMISD_CALL_NOLINK_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
102 if (RetVT.SimpleTy != MVT::isVoid)
103 return 0;
104 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
105 return fastEmitInst_r(ARM::tBX_CALL, &ARM::tGPRRegClass, Op0, Op0IsKill);
106 }
107 if ((!Subtarget->isThumb()) && (!Subtarget->hasV4TOps())) {
108 return fastEmitInst_r(ARM::BMOVPCRX_CALL, &ARM::tGPRRegClass, Op0, Op0IsKill);
109 }
110 if ((Subtarget->hasV4TOps()) && (!Subtarget->isThumb())) {
111 return fastEmitInst_r(ARM::BX_CALL, &ARM::tGPRRegClass, Op0, Op0IsKill);
112 }
113 return 0;
114}
115
116unsigned fastEmit_ARMISD_CALL_NOLINK_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
117 switch (VT.SimpleTy) {
118 case MVT::i32: return fastEmit_ARMISD_CALL_NOLINK_MVT_i32_r(RetVT, Op0, Op0IsKill);
119 default: return 0;
120 }
121}
122
123// FastEmit functions for ARMISD::CALL_PRED.
124
125unsigned fastEmit_ARMISD_CALL_PRED_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
126 if (RetVT.SimpleTy != MVT::isVoid)
127 return 0;
128 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
129 return fastEmitInst_r(ARM::BLX_pred_noip, &ARM::GPRnoipRegClass, Op0, Op0IsKill);
130 }
131 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( !MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
132 return fastEmitInst_r(ARM::BLX_pred, &ARM::GPRRegClass, Op0, Op0IsKill);
133 }
134 return 0;
135}
136
137unsigned fastEmit_ARMISD_CALL_PRED_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
138 switch (VT.SimpleTy) {
139 case MVT::i32: return fastEmit_ARMISD_CALL_PRED_MVT_i32_r(RetVT, Op0, Op0IsKill);
140 default: return 0;
141 }
142}
143
144// FastEmit functions for ARMISD::CMPFPEw0.
145
146unsigned fastEmit_ARMISD_CMPFPEw0_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
147 if (RetVT.SimpleTy != MVT::isVoid)
148 return 0;
149 if ((Subtarget->hasFullFP16())) {
150 return fastEmitInst_r(ARM::VCMPEZH, &ARM::HPRRegClass, Op0, Op0IsKill);
151 }
152 return 0;
153}
154
155unsigned fastEmit_ARMISD_CMPFPEw0_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
156 if (RetVT.SimpleTy != MVT::isVoid)
157 return 0;
158 if ((Subtarget->hasVFP2Base())) {
159 return fastEmitInst_r(ARM::VCMPEZS, &ARM::SPRRegClass, Op0, Op0IsKill);
160 }
161 return 0;
162}
163
164unsigned fastEmit_ARMISD_CMPFPEw0_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
165 if (RetVT.SimpleTy != MVT::isVoid)
166 return 0;
167 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
168 return fastEmitInst_r(ARM::VCMPEZD, &ARM::DPRRegClass, Op0, Op0IsKill);
169 }
170 return 0;
171}
172
173unsigned fastEmit_ARMISD_CMPFPEw0_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
174 switch (VT.SimpleTy) {
175 case MVT::f16: return fastEmit_ARMISD_CMPFPEw0_MVT_f16_r(RetVT, Op0, Op0IsKill);
176 case MVT::f32: return fastEmit_ARMISD_CMPFPEw0_MVT_f32_r(RetVT, Op0, Op0IsKill);
177 case MVT::f64: return fastEmit_ARMISD_CMPFPEw0_MVT_f64_r(RetVT, Op0, Op0IsKill);
178 default: return 0;
179 }
180}
181
182// FastEmit functions for ARMISD::CMPFPw0.
183
184unsigned fastEmit_ARMISD_CMPFPw0_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
185 if (RetVT.SimpleTy != MVT::isVoid)
186 return 0;
187 if ((Subtarget->hasFullFP16())) {
188 return fastEmitInst_r(ARM::VCMPZH, &ARM::HPRRegClass, Op0, Op0IsKill);
189 }
190 return 0;
191}
192
193unsigned fastEmit_ARMISD_CMPFPw0_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
194 if (RetVT.SimpleTy != MVT::isVoid)
195 return 0;
196 if ((Subtarget->hasVFP2Base())) {
197 return fastEmitInst_r(ARM::VCMPZS, &ARM::SPRRegClass, Op0, Op0IsKill);
198 }
199 return 0;
200}
201
202unsigned fastEmit_ARMISD_CMPFPw0_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
203 if (RetVT.SimpleTy != MVT::isVoid)
204 return 0;
205 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
206 return fastEmitInst_r(ARM::VCMPZD, &ARM::DPRRegClass, Op0, Op0IsKill);
207 }
208 return 0;
209}
210
211unsigned fastEmit_ARMISD_CMPFPw0_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
212 switch (VT.SimpleTy) {
213 case MVT::f16: return fastEmit_ARMISD_CMPFPw0_MVT_f16_r(RetVT, Op0, Op0IsKill);
214 case MVT::f32: return fastEmit_ARMISD_CMPFPw0_MVT_f32_r(RetVT, Op0, Op0IsKill);
215 case MVT::f64: return fastEmit_ARMISD_CMPFPw0_MVT_f64_r(RetVT, Op0, Op0IsKill);
216 default: return 0;
217 }
218}
219
220// FastEmit functions for ARMISD::RRX.
221
222unsigned fastEmit_ARMISD_RRX_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
223 if (RetVT.SimpleTy != MVT::i32)
224 return 0;
225 if ((Subtarget->isThumb2())) {
226 return fastEmitInst_r(ARM::t2RRX, &ARM::rGPRRegClass, Op0, Op0IsKill);
227 }
228 if ((!Subtarget->isThumb())) {
229 return fastEmitInst_r(ARM::RRX, &ARM::GPRRegClass, Op0, Op0IsKill);
230 }
231 return 0;
232}
233
234unsigned fastEmit_ARMISD_RRX_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
235 switch (VT.SimpleTy) {
236 case MVT::i32: return fastEmit_ARMISD_RRX_MVT_i32_r(RetVT, Op0, Op0IsKill);
237 default: return 0;
238 }
239}
240
241// FastEmit functions for ARMISD::SRA_FLAG.
242
243unsigned fastEmit_ARMISD_SRA_FLAG_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
244 if (RetVT.SimpleTy != MVT::i32)
245 return 0;
246 if ((Subtarget->isThumb2())) {
247 return fastEmitInst_r(ARM::t2MOVsra_flag, &ARM::rGPRRegClass, Op0, Op0IsKill);
248 }
249 if ((!Subtarget->isThumb())) {
250 return fastEmitInst_r(ARM::MOVsra_flag, &ARM::GPRRegClass, Op0, Op0IsKill);
251 }
252 return 0;
253}
254
255unsigned fastEmit_ARMISD_SRA_FLAG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
256 switch (VT.SimpleTy) {
257 case MVT::i32: return fastEmit_ARMISD_SRA_FLAG_MVT_i32_r(RetVT, Op0, Op0IsKill);
258 default: return 0;
259 }
260}
261
262// FastEmit functions for ARMISD::SRL_FLAG.
263
264unsigned fastEmit_ARMISD_SRL_FLAG_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
265 if (RetVT.SimpleTy != MVT::i32)
266 return 0;
267 if ((Subtarget->isThumb2())) {
268 return fastEmitInst_r(ARM::t2MOVsrl_flag, &ARM::rGPRRegClass, Op0, Op0IsKill);
269 }
270 if ((!Subtarget->isThumb())) {
271 return fastEmitInst_r(ARM::MOVsrl_flag, &ARM::GPRRegClass, Op0, Op0IsKill);
272 }
273 return 0;
274}
275
276unsigned fastEmit_ARMISD_SRL_FLAG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
277 switch (VT.SimpleTy) {
278 case MVT::i32: return fastEmit_ARMISD_SRL_FLAG_MVT_i32_r(RetVT, Op0, Op0IsKill);
279 default: return 0;
280 }
281}
282
283// FastEmit functions for ARMISD::TC_RETURN.
284
285unsigned fastEmit_ARMISD_TC_RETURN_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
286 if (RetVT.SimpleTy != MVT::isVoid)
287 return 0;
288 return fastEmitInst_r(ARM::TCRETURNri, &ARM::tcGPRRegClass, Op0, Op0IsKill);
289}
290
291unsigned fastEmit_ARMISD_TC_RETURN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
292 switch (VT.SimpleTy) {
293 case MVT::i32: return fastEmit_ARMISD_TC_RETURN_MVT_i32_r(RetVT, Op0, Op0IsKill);
294 default: return 0;
295 }
296}
297
298// FastEmit functions for ARMISD::VADDVs.
299
300unsigned fastEmit_ARMISD_VADDVs_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
301 if (RetVT.SimpleTy != MVT::i32)
302 return 0;
303 if ((Subtarget->hasMVEIntegerOps())) {
304 return fastEmitInst_r(ARM::MVE_VADDVs8no_acc, &ARM::tGPREvenRegClass, Op0, Op0IsKill);
305 }
306 return 0;
307}
308
309unsigned fastEmit_ARMISD_VADDVs_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
310 if (RetVT.SimpleTy != MVT::i32)
311 return 0;
312 if ((Subtarget->hasMVEIntegerOps())) {
313 return fastEmitInst_r(ARM::MVE_VADDVs16no_acc, &ARM::tGPREvenRegClass, Op0, Op0IsKill);
314 }
315 return 0;
316}
317
318unsigned fastEmit_ARMISD_VADDVs_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
319 if (RetVT.SimpleTy != MVT::i32)
320 return 0;
321 if ((Subtarget->hasMVEIntegerOps())) {
322 return fastEmitInst_r(ARM::MVE_VADDVs32no_acc, &ARM::tGPREvenRegClass, Op0, Op0IsKill);
323 }
324 return 0;
325}
326
327unsigned fastEmit_ARMISD_VADDVs_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
328 switch (VT.SimpleTy) {
329 case MVT::v16i8: return fastEmit_ARMISD_VADDVs_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
330 case MVT::v8i16: return fastEmit_ARMISD_VADDVs_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
331 case MVT::v4i32: return fastEmit_ARMISD_VADDVs_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
332 default: return 0;
333 }
334}
335
336// FastEmit functions for ARMISD::VADDVu.
337
338unsigned fastEmit_ARMISD_VADDVu_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
339 if (RetVT.SimpleTy != MVT::i32)
340 return 0;
341 if ((Subtarget->hasMVEIntegerOps())) {
342 return fastEmitInst_r(ARM::MVE_VADDVu8no_acc, &ARM::tGPREvenRegClass, Op0, Op0IsKill);
343 }
344 return 0;
345}
346
347unsigned fastEmit_ARMISD_VADDVu_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
348 if (RetVT.SimpleTy != MVT::i32)
349 return 0;
350 if ((Subtarget->hasMVEIntegerOps())) {
351 return fastEmitInst_r(ARM::MVE_VADDVu16no_acc, &ARM::tGPREvenRegClass, Op0, Op0IsKill);
352 }
353 return 0;
354}
355
356unsigned fastEmit_ARMISD_VADDVu_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
357 if (RetVT.SimpleTy != MVT::i32)
358 return 0;
359 if ((Subtarget->hasMVEIntegerOps())) {
360 return fastEmitInst_r(ARM::MVE_VADDVu32no_acc, &ARM::tGPREvenRegClass, Op0, Op0IsKill);
361 }
362 return 0;
363}
364
365unsigned fastEmit_ARMISD_VADDVu_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
366 switch (VT.SimpleTy) {
367 case MVT::v16i8: return fastEmit_ARMISD_VADDVu_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
368 case MVT::v8i16: return fastEmit_ARMISD_VADDVu_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
369 case MVT::v4i32: return fastEmit_ARMISD_VADDVu_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
370 default: return 0;
371 }
372}
373
374// FastEmit functions for ARMISD::VDUP.
375
376unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
377 if ((Subtarget->hasNEON())) {
378 return fastEmitInst_r(ARM::VDUP8d, &ARM::DPRRegClass, Op0, Op0IsKill);
379 }
380 return 0;
381}
382
383unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
384 if ((Subtarget->hasMVEIntegerOps())) {
385 return fastEmitInst_r(ARM::MVE_VDUP8, &ARM::MQPRRegClass, Op0, Op0IsKill);
386 }
387 if ((Subtarget->hasNEON())) {
388 return fastEmitInst_r(ARM::VDUP8q, &ARM::QPRRegClass, Op0, Op0IsKill);
389 }
390 return 0;
391}
392
393unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
394 if ((Subtarget->hasNEON())) {
395 return fastEmitInst_r(ARM::VDUP16d, &ARM::DPRRegClass, Op0, Op0IsKill);
396 }
397 return 0;
398}
399
400unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
401 if ((Subtarget->hasMVEIntegerOps())) {
402 return fastEmitInst_r(ARM::MVE_VDUP16, &ARM::MQPRRegClass, Op0, Op0IsKill);
403 }
404 if ((Subtarget->hasNEON())) {
405 return fastEmitInst_r(ARM::VDUP16q, &ARM::QPRRegClass, Op0, Op0IsKill);
406 }
407 return 0;
408}
409
410unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
411 if ((!Subtarget->hasSlowVDUP32()) && (Subtarget->hasNEON())) {
412 return fastEmitInst_r(ARM::VDUP32d, &ARM::DPRRegClass, Op0, Op0IsKill);
413 }
414 return 0;
415}
416
417unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
418 if ((Subtarget->hasMVEIntegerOps())) {
419 return fastEmitInst_r(ARM::MVE_VDUP32, &ARM::MQPRRegClass, Op0, Op0IsKill);
420 }
421 if ((Subtarget->hasNEON())) {
422 return fastEmitInst_r(ARM::VDUP32q, &ARM::QPRRegClass, Op0, Op0IsKill);
423 }
424 return 0;
425}
426
427unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
428 if ((Subtarget->hasMVEIntegerOps())) {
429 return fastEmitInst_r(ARM::MVE_VDUP16, &ARM::MQPRRegClass, Op0, Op0IsKill);
430 }
431 return 0;
432}
433
434unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
435 if ((Subtarget->hasMVEIntegerOps())) {
436 return fastEmitInst_r(ARM::MVE_VDUP32, &ARM::MQPRRegClass, Op0, Op0IsKill);
437 }
438 return 0;
439}
440
441unsigned fastEmit_ARMISD_VDUP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
442switch (RetVT.SimpleTy) {
443 case MVT::v8i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(Op0, Op0IsKill);
444 case MVT::v16i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(Op0, Op0IsKill);
445 case MVT::v4i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(Op0, Op0IsKill);
446 case MVT::v8i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(Op0, Op0IsKill);
447 case MVT::v2i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(Op0, Op0IsKill);
448 case MVT::v4i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
449 case MVT::v8f16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(Op0, Op0IsKill);
450 case MVT::v4f32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(Op0, Op0IsKill);
451 default: return 0;
452}
453}
454
455unsigned fastEmit_ARMISD_VDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
456 switch (VT.SimpleTy) {
457 case MVT::i32: return fastEmit_ARMISD_VDUP_MVT_i32_r(RetVT, Op0, Op0IsKill);
458 default: return 0;
459 }
460}
461
462// FastEmit functions for ARMISD::VMOVSR.
463
464unsigned fastEmit_ARMISD_VMOVSR_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
465 if (RetVT.SimpleTy != MVT::f32)
466 return 0;
467 if ((Subtarget->hasVFP2Base()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
468 return fastEmitInst_r(ARM::VMOVSR, &ARM::SPRRegClass, Op0, Op0IsKill);
469 }
470 return 0;
471}
472
473unsigned fastEmit_ARMISD_VMOVSR_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
474 switch (VT.SimpleTy) {
475 case MVT::i32: return fastEmit_ARMISD_VMOVSR_MVT_i32_r(RetVT, Op0, Op0IsKill);
476 default: return 0;
477 }
478}
479
480// FastEmit functions for ARMISD::VMOVhr.
481
482unsigned fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(unsigned Op0, bool Op0IsKill) {
483 if ((Subtarget->hasFPRegs16())) {
484 return fastEmitInst_r(ARM::VMOVHR, &ARM::HPRRegClass, Op0, Op0IsKill);
485 }
486 return 0;
487}
488
489unsigned fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
490 if ((Subtarget->hasFPRegs16())) {
491 return fastEmitInst_r(ARM::VMOVHR, &ARM::HPRRegClass, Op0, Op0IsKill);
492 }
493 return 0;
494}
495
496unsigned fastEmit_ARMISD_VMOVhr_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
497switch (RetVT.SimpleTy) {
498 case MVT::bf16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(Op0, Op0IsKill);
499 case MVT::f16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(Op0, Op0IsKill);
500 default: return 0;
501}
502}
503
504unsigned fastEmit_ARMISD_VMOVhr_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
505 switch (VT.SimpleTy) {
506 case MVT::i32: return fastEmit_ARMISD_VMOVhr_MVT_i32_r(RetVT, Op0, Op0IsKill);
507 default: return 0;
508 }
509}
510
511// FastEmit functions for ARMISD::VMOVrh.
512
513unsigned fastEmit_ARMISD_VMOVrh_MVT_bf16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
514 if (RetVT.SimpleTy != MVT::i32)
515 return 0;
516 if ((Subtarget->hasFPRegs16())) {
517 return fastEmitInst_r(ARM::VMOVRH, &ARM::rGPRRegClass, Op0, Op0IsKill);
518 }
519 return 0;
520}
521
522unsigned fastEmit_ARMISD_VMOVrh_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
523 if (RetVT.SimpleTy != MVT::i32)
524 return 0;
525 if ((Subtarget->hasFPRegs16())) {
526 return fastEmitInst_r(ARM::VMOVRH, &ARM::rGPRRegClass, Op0, Op0IsKill);
527 }
528 return 0;
529}
530
531unsigned fastEmit_ARMISD_VMOVrh_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
532 switch (VT.SimpleTy) {
533 case MVT::bf16: return fastEmit_ARMISD_VMOVrh_MVT_bf16_r(RetVT, Op0, Op0IsKill);
534 case MVT::f16: return fastEmit_ARMISD_VMOVrh_MVT_f16_r(RetVT, Op0, Op0IsKill);
535 default: return 0;
536 }
537}
538
539// FastEmit functions for ARMISD::VREV16.
540
541unsigned fastEmit_ARMISD_VREV16_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
542 if (RetVT.SimpleTy != MVT::v8i8)
543 return 0;
544 if ((Subtarget->hasNEON())) {
545 return fastEmitInst_r(ARM::VREV16d8, &ARM::DPRRegClass, Op0, Op0IsKill);
546 }
547 return 0;
548}
549
550unsigned fastEmit_ARMISD_VREV16_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
551 if (RetVT.SimpleTy != MVT::v16i8)
552 return 0;
553 if ((Subtarget->hasMVEIntegerOps())) {
554 return fastEmitInst_r(ARM::MVE_VREV16_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
555 }
556 if ((Subtarget->hasNEON())) {
557 return fastEmitInst_r(ARM::VREV16q8, &ARM::QPRRegClass, Op0, Op0IsKill);
558 }
559 return 0;
560}
561
562unsigned fastEmit_ARMISD_VREV16_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
563 switch (VT.SimpleTy) {
564 case MVT::v8i8: return fastEmit_ARMISD_VREV16_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
565 case MVT::v16i8: return fastEmit_ARMISD_VREV16_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
566 default: return 0;
567 }
568}
569
570// FastEmit functions for ARMISD::VREV32.
571
572unsigned fastEmit_ARMISD_VREV32_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
573 if (RetVT.SimpleTy != MVT::v8i8)
574 return 0;
575 if ((Subtarget->hasNEON())) {
576 return fastEmitInst_r(ARM::VREV32d8, &ARM::DPRRegClass, Op0, Op0IsKill);
577 }
578 return 0;
579}
580
581unsigned fastEmit_ARMISD_VREV32_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
582 if (RetVT.SimpleTy != MVT::v16i8)
583 return 0;
584 if ((Subtarget->hasMVEIntegerOps())) {
585 return fastEmitInst_r(ARM::MVE_VREV32_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
586 }
587 if ((Subtarget->hasNEON())) {
588 return fastEmitInst_r(ARM::VREV32q8, &ARM::QPRRegClass, Op0, Op0IsKill);
589 }
590 return 0;
591}
592
593unsigned fastEmit_ARMISD_VREV32_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
594 if (RetVT.SimpleTy != MVT::v4i16)
595 return 0;
596 if ((Subtarget->hasNEON())) {
597 return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0, Op0IsKill);
598 }
599 return 0;
600}
601
602unsigned fastEmit_ARMISD_VREV32_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
603 if (RetVT.SimpleTy != MVT::v8i16)
604 return 0;
605 if ((Subtarget->hasMVEIntegerOps())) {
606 return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
607 }
608 if ((Subtarget->hasNEON())) {
609 return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0, Op0IsKill);
610 }
611 return 0;
612}
613
614unsigned fastEmit_ARMISD_VREV32_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
615 if (RetVT.SimpleTy != MVT::v4f16)
616 return 0;
617 if ((Subtarget->hasNEON())) {
618 return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0, Op0IsKill);
619 }
620 return 0;
621}
622
623unsigned fastEmit_ARMISD_VREV32_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
624 if (RetVT.SimpleTy != MVT::v8f16)
625 return 0;
626 if ((Subtarget->hasMVEIntegerOps())) {
627 return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
628 }
629 if ((Subtarget->hasNEON())) {
630 return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0, Op0IsKill);
631 }
632 return 0;
633}
634
635unsigned fastEmit_ARMISD_VREV32_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
636 switch (VT.SimpleTy) {
637 case MVT::v8i8: return fastEmit_ARMISD_VREV32_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
638 case MVT::v16i8: return fastEmit_ARMISD_VREV32_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
639 case MVT::v4i16: return fastEmit_ARMISD_VREV32_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
640 case MVT::v8i16: return fastEmit_ARMISD_VREV32_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
641 case MVT::v4f16: return fastEmit_ARMISD_VREV32_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
642 case MVT::v8f16: return fastEmit_ARMISD_VREV32_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
643 default: return 0;
644 }
645}
646
647// FastEmit functions for ARMISD::VREV64.
648
649unsigned fastEmit_ARMISD_VREV64_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
650 if (RetVT.SimpleTy != MVT::v8i8)
651 return 0;
652 if ((Subtarget->hasNEON())) {
653 return fastEmitInst_r(ARM::VREV64d8, &ARM::DPRRegClass, Op0, Op0IsKill);
654 }
655 return 0;
656}
657
658unsigned fastEmit_ARMISD_VREV64_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
659 if (RetVT.SimpleTy != MVT::v16i8)
660 return 0;
661 if ((Subtarget->hasMVEIntegerOps())) {
662 return fastEmitInst_r(ARM::MVE_VREV64_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
663 }
664 if ((Subtarget->hasNEON())) {
665 return fastEmitInst_r(ARM::VREV64q8, &ARM::QPRRegClass, Op0, Op0IsKill);
666 }
667 return 0;
668}
669
670unsigned fastEmit_ARMISD_VREV64_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
671 if (RetVT.SimpleTy != MVT::v4i16)
672 return 0;
673 if ((Subtarget->hasNEON())) {
674 return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0, Op0IsKill);
675 }
676 return 0;
677}
678
679unsigned fastEmit_ARMISD_VREV64_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
680 if (RetVT.SimpleTy != MVT::v8i16)
681 return 0;
682 if ((Subtarget->hasMVEIntegerOps())) {
683 return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
684 }
685 if ((Subtarget->hasNEON())) {
686 return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0, Op0IsKill);
687 }
688 return 0;
689}
690
691unsigned fastEmit_ARMISD_VREV64_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
692 if (RetVT.SimpleTy != MVT::v2i32)
693 return 0;
694 if ((Subtarget->hasNEON())) {
695 return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0, Op0IsKill);
696 }
697 return 0;
698}
699
700unsigned fastEmit_ARMISD_VREV64_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
701 if (RetVT.SimpleTy != MVT::v4i32)
702 return 0;
703 if ((Subtarget->hasMVEIntegerOps())) {
704 return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0, Op0IsKill);
705 }
706 if ((Subtarget->hasNEON())) {
707 return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0, Op0IsKill);
708 }
709 return 0;
710}
711
712unsigned fastEmit_ARMISD_VREV64_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
713 if (RetVT.SimpleTy != MVT::v4f16)
714 return 0;
715 if ((Subtarget->hasNEON())) {
716 return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0, Op0IsKill);
717 }
718 return 0;
719}
720
721unsigned fastEmit_ARMISD_VREV64_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
722 if (RetVT.SimpleTy != MVT::v8f16)
723 return 0;
724 if ((Subtarget->hasMVEIntegerOps())) {
725 return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
726 }
727 if ((Subtarget->hasNEON())) {
728 return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0, Op0IsKill);
729 }
730 return 0;
731}
732
733unsigned fastEmit_ARMISD_VREV64_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
734 if (RetVT.SimpleTy != MVT::v2f32)
735 return 0;
736 if ((Subtarget->hasNEON())) {
737 return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0, Op0IsKill);
738 }
739 return 0;
740}
741
742unsigned fastEmit_ARMISD_VREV64_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
743 if (RetVT.SimpleTy != MVT::v4f32)
744 return 0;
745 if ((Subtarget->hasMVEIntegerOps())) {
746 return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0, Op0IsKill);
747 }
748 if ((Subtarget->hasNEON())) {
749 return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0, Op0IsKill);
750 }
751 return 0;
752}
753
754unsigned fastEmit_ARMISD_VREV64_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
755 switch (VT.SimpleTy) {
756 case MVT::v8i8: return fastEmit_ARMISD_VREV64_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
757 case MVT::v16i8: return fastEmit_ARMISD_VREV64_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
758 case MVT::v4i16: return fastEmit_ARMISD_VREV64_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
759 case MVT::v8i16: return fastEmit_ARMISD_VREV64_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
760 case MVT::v2i32: return fastEmit_ARMISD_VREV64_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
761 case MVT::v4i32: return fastEmit_ARMISD_VREV64_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
762 case MVT::v4f16: return fastEmit_ARMISD_VREV64_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
763 case MVT::v8f16: return fastEmit_ARMISD_VREV64_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
764 case MVT::v2f32: return fastEmit_ARMISD_VREV64_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
765 case MVT::v4f32: return fastEmit_ARMISD_VREV64_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
766 default: return 0;
767 }
768}
769
770// FastEmit functions for ARMISD::WIN__DBZCHK.
771
772unsigned fastEmit_ARMISD_WIN__DBZCHK_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
773 if (RetVT.SimpleTy != MVT::isVoid)
774 return 0;
775 return fastEmitInst_r(ARM::WIN__DBZCHK, &ARM::tGPRRegClass, Op0, Op0IsKill);
776}
777
778unsigned fastEmit_ARMISD_WIN__DBZCHK_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
779 switch (VT.SimpleTy) {
780 case MVT::i32: return fastEmit_ARMISD_WIN__DBZCHK_MVT_i32_r(RetVT, Op0, Op0IsKill);
781 default: return 0;
782 }
783}
784
785// FastEmit functions for ARMISD::tSECALL.
786
787unsigned fastEmit_ARMISD_tSECALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
788 if (RetVT.SimpleTy != MVT::isVoid)
789 return 0;
790 if ((Subtarget->has8MSecExt()) && (Subtarget->isThumb())) {
791 return fastEmitInst_r(ARM::tBLXNS_CALL, &ARM::GPRnopcRegClass, Op0, Op0IsKill);
792 }
793 return 0;
794}
795
796unsigned fastEmit_ARMISD_tSECALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
797 switch (VT.SimpleTy) {
798 case MVT::i32: return fastEmit_ARMISD_tSECALL_MVT_i32_r(RetVT, Op0, Op0IsKill);
799 default: return 0;
800 }
801}
802
803// FastEmit functions for ISD::ABS.
804
805unsigned fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
806 if (RetVT.SimpleTy != MVT::v8i8)
807 return 0;
808 if ((Subtarget->hasNEON())) {
809 return fastEmitInst_r(ARM::VABSv8i8, &ARM::DPRRegClass, Op0, Op0IsKill);
810 }
811 return 0;
812}
813
814unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
815 if (RetVT.SimpleTy != MVT::v16i8)
816 return 0;
817 if ((Subtarget->hasMVEIntegerOps())) {
818 return fastEmitInst_r(ARM::MVE_VABSs8, &ARM::MQPRRegClass, Op0, Op0IsKill);
819 }
820 if ((Subtarget->hasNEON())) {
821 return fastEmitInst_r(ARM::VABSv16i8, &ARM::QPRRegClass, Op0, Op0IsKill);
822 }
823 return 0;
824}
825
826unsigned fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
827 if (RetVT.SimpleTy != MVT::v4i16)
828 return 0;
829 if ((Subtarget->hasNEON())) {
830 return fastEmitInst_r(ARM::VABSv4i16, &ARM::DPRRegClass, Op0, Op0IsKill);
831 }
832 return 0;
833}
834
835unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
836 if (RetVT.SimpleTy != MVT::v8i16)
837 return 0;
838 if ((Subtarget->hasMVEIntegerOps())) {
839 return fastEmitInst_r(ARM::MVE_VABSs16, &ARM::MQPRRegClass, Op0, Op0IsKill);
840 }
841 if ((Subtarget->hasNEON())) {
842 return fastEmitInst_r(ARM::VABSv8i16, &ARM::QPRRegClass, Op0, Op0IsKill);
843 }
844 return 0;
845}
846
847unsigned fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
848 if (RetVT.SimpleTy != MVT::v2i32)
849 return 0;
850 if ((Subtarget->hasNEON())) {
851 return fastEmitInst_r(ARM::VABSv2i32, &ARM::DPRRegClass, Op0, Op0IsKill);
852 }
853 return 0;
854}
855
856unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
857 if (RetVT.SimpleTy != MVT::v4i32)
858 return 0;
859 if ((Subtarget->hasMVEIntegerOps())) {
860 return fastEmitInst_r(ARM::MVE_VABSs32, &ARM::MQPRRegClass, Op0, Op0IsKill);
861 }
862 if ((Subtarget->hasNEON())) {
863 return fastEmitInst_r(ARM::VABSv4i32, &ARM::QPRRegClass, Op0, Op0IsKill);
864 }
865 return 0;
866}
867
868unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
869 switch (VT.SimpleTy) {
870 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
871 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
872 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
873 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
874 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
875 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
876 default: return 0;
877 }
878}
879
880// FastEmit functions for ISD::ANY_EXTEND.
881
882unsigned fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
883 if (RetVT.SimpleTy != MVT::v8i16)
884 return 0;
885 if ((Subtarget->hasNEON())) {
886 return fastEmitInst_r(ARM::VMOVLuv8i16, &ARM::QPRRegClass, Op0, Op0IsKill);
887 }
888 return 0;
889}
890
891unsigned fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
892 if (RetVT.SimpleTy != MVT::v4i32)
893 return 0;
894 if ((Subtarget->hasNEON())) {
895 return fastEmitInst_r(ARM::VMOVLuv4i32, &ARM::QPRRegClass, Op0, Op0IsKill);
896 }
897 return 0;
898}
899
900unsigned fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
901 if (RetVT.SimpleTy != MVT::v2i64)
902 return 0;
903 if ((Subtarget->hasNEON())) {
904 return fastEmitInst_r(ARM::VMOVLuv2i64, &ARM::QPRRegClass, Op0, Op0IsKill);
905 }
906 return 0;
907}
908
909unsigned fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
910 switch (VT.SimpleTy) {
911 case MVT::v8i8: return fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
912 case MVT::v4i16: return fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
913 case MVT::v2i32: return fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
914 default: return 0;
915 }
916}
917
918// FastEmit functions for ISD::BITCAST.
919
920unsigned fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
921 if (RetVT.SimpleTy != MVT::f32)
922 return 0;
923 if ((Subtarget->hasFPRegs()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
924 return fastEmitInst_r(ARM::VMOVSR, &ARM::SPRRegClass, Op0, Op0IsKill);
925 }
926 return 0;
927}
928
929unsigned fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
930 if (RetVT.SimpleTy != MVT::i32)
931 return 0;
932 if ((Subtarget->hasFPRegs())) {
933 return fastEmitInst_r(ARM::VMOVRS, &ARM::GPRRegClass, Op0, Op0IsKill);
934 }
935 return 0;
936}
937
938unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
939 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
940 return fastEmitInst_r(ARM::VREV64d8, &ARM::DPRRegClass, Op0, Op0IsKill);
941 }
942 return 0;
943}
944
945unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
946 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
947 return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0, Op0IsKill);
948 }
949 return 0;
950}
951
952unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
953 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
954 return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0, Op0IsKill);
955 }
956 return 0;
957}
958
959unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
960 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
961 return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0, Op0IsKill);
962 }
963 return 0;
964}
965
966unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(unsigned Op0, bool Op0IsKill) {
967 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
968 return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0, Op0IsKill);
969 }
970 return 0;
971}
972
973unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
974 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
975 return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0, Op0IsKill);
976 }
977 return 0;
978}
979
980unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
981switch (RetVT.SimpleTy) {
982 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0, Op0IsKill);
983 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0, Op0IsKill);
984 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0, Op0IsKill);
985 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0, Op0IsKill);
986 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0, Op0IsKill);
987 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0, Op0IsKill);
988 default: return 0;
989}
990}
991
992unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
993 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
994 return fastEmitInst_r(ARM::VREV64d8, &ARM::DPRRegClass, Op0, Op0IsKill);
995 }
996 return 0;
997}
998
999unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
1000 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1001 return fastEmitInst_r(ARM::VREV16d8, &ARM::DPRRegClass, Op0, Op0IsKill);
1002 }
1003 return 0;
1004}
1005
1006unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
1007 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1008 return fastEmitInst_r(ARM::VREV32d8, &ARM::DPRRegClass, Op0, Op0IsKill);
1009 }
1010 return 0;
1011}
1012
1013unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
1014 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1015 return fastEmitInst_r(ARM::VREV64d8, &ARM::DPRRegClass, Op0, Op0IsKill);
1016 }
1017 return 0;
1018}
1019
1020unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
1021 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1022 return fastEmitInst_r(ARM::VREV16d8, &ARM::DPRRegClass, Op0, Op0IsKill);
1023 }
1024 return 0;
1025}
1026
1027unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(unsigned Op0, bool Op0IsKill) {
1028 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1029 return fastEmitInst_r(ARM::VREV16d8, &ARM::DPRRegClass, Op0, Op0IsKill);
1030 }
1031 return 0;
1032}
1033
1034unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
1035 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1036 return fastEmitInst_r(ARM::VREV32d8, &ARM::DPRRegClass, Op0, Op0IsKill);
1037 }
1038 return 0;
1039}
1040
1041unsigned fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1042switch (RetVT.SimpleTy) {
1043 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0, Op0IsKill);
1044 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0, Op0IsKill);
1045 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0, Op0IsKill);
1046 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0, Op0IsKill);
1047 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0, Op0IsKill);
1048 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0, Op0IsKill);
1049 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0, Op0IsKill);
1050 default: return 0;
1051}
1052}
1053
1054unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
1055 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1056 return fastEmitInst_r(ARM::MVE_VREV16_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
1057 }
1058 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1059 return fastEmitInst_r(ARM::VREV16q8, &ARM::QPRRegClass, Op0, Op0IsKill);
1060 }
1061 return 0;
1062}
1063
1064unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1065 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1066 return fastEmitInst_r(ARM::MVE_VREV32_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
1067 }
1068 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1069 return fastEmitInst_r(ARM::VREV32q8, &ARM::QPRRegClass, Op0, Op0IsKill);
1070 }
1071 return 0;
1072}
1073
1074unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
1075 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1076 return fastEmitInst_r(ARM::MVE_VREV64_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
1077 }
1078 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1079 return fastEmitInst_r(ARM::VREV64q8, &ARM::QPRRegClass, Op0, Op0IsKill);
1080 }
1081 return 0;
1082}
1083
1084unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
1085 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1086 return fastEmitInst_r(ARM::MVE_VREV16_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
1087 }
1088 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1089 return fastEmitInst_r(ARM::VREV16q8, &ARM::QPRRegClass, Op0, Op0IsKill);
1090 }
1091 return 0;
1092}
1093
1094unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(unsigned Op0, bool Op0IsKill) {
1095 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1096 return fastEmitInst_r(ARM::VREV16q8, &ARM::QPRRegClass, Op0, Op0IsKill);
1097 }
1098 return 0;
1099}
1100
1101unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
1102 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1103 return fastEmitInst_r(ARM::MVE_VREV32_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
1104 }
1105 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1106 return fastEmitInst_r(ARM::VREV32q8, &ARM::QPRRegClass, Op0, Op0IsKill);
1107 }
1108 return 0;
1109}
1110
1111unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
1112 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1113 return fastEmitInst_r(ARM::MVE_VREV64_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
1114 }
1115 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1116 return fastEmitInst_r(ARM::VREV64q8, &ARM::QPRRegClass, Op0, Op0IsKill);
1117 }
1118 return 0;
1119}
1120
1121unsigned fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1122switch (RetVT.SimpleTy) {
1123 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
1124 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
1125 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
1126 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0, Op0IsKill);
1127 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0, Op0IsKill);
1128 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0, Op0IsKill);
1129 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0, Op0IsKill);
1130 default: return 0;
1131}
1132}
1133
1134unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
1135 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1136 return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1137 }
1138 return 0;
1139}
1140
1141unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
1142 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1143 return fastEmitInst_r(ARM::VREV16d8, &ARM::DPRRegClass, Op0, Op0IsKill);
1144 }
1145 return 0;
1146}
1147
1148unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
1149 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1150 return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1151 }
1152 return 0;
1153}
1154
1155unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
1156 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1157 return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1158 }
1159 return 0;
1160}
1161
1162unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
1163 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1164 return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1165 }
1166 return 0;
1167}
1168
1169unsigned fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1170switch (RetVT.SimpleTy) {
1171 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0, Op0IsKill);
1172 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0, Op0IsKill);
1173 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0, Op0IsKill);
1174 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0, Op0IsKill);
1175 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0, Op0IsKill);
1176 default: return 0;
1177}
1178}
1179
1180unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
1181 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1182 return fastEmitInst_r(ARM::MVE_VREV16_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
1183 }
1184 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1185 return fastEmitInst_r(ARM::VREV16q8, &ARM::QPRRegClass, Op0, Op0IsKill);
1186 }
1187 return 0;
1188}
1189
1190unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1191 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1192 return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
1193 }
1194 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1195 return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1196 }
1197 return 0;
1198}
1199
1200unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
1201 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1202 return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
1203 }
1204 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1205 return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1206 }
1207 return 0;
1208}
1209
1210unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
1211 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1212 return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
1213 }
1214 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1215 return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1216 }
1217 return 0;
1218}
1219
1220unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
1221 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1222 return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
1223 }
1224 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1225 return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1226 }
1227 return 0;
1228}
1229
1230unsigned fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1231switch (RetVT.SimpleTy) {
1232 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0, Op0IsKill);
1233 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
1234 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
1235 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0, Op0IsKill);
1236 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0, Op0IsKill);
1237 default: return 0;
1238}
1239}
1240
1241unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
1242 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1243 return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0, Op0IsKill);
1244 }
1245 return 0;
1246}
1247
1248unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
1249 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1250 return fastEmitInst_r(ARM::VREV32d8, &ARM::DPRRegClass, Op0, Op0IsKill);
1251 }
1252 return 0;
1253}
1254
1255unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
1256 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1257 return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1258 }
1259 return 0;
1260}
1261
1262unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
1263 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1264 return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0, Op0IsKill);
1265 }
1266 return 0;
1267}
1268
1269unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
1270 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1271 return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1272 }
1273 return 0;
1274}
1275
1276unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(unsigned Op0, bool Op0IsKill) {
1277 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1278 return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1279 }
1280 return 0;
1281}
1282
1283unsigned fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1284switch (RetVT.SimpleTy) {
1285 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0, Op0IsKill);
1286 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0, Op0IsKill);
1287 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0, Op0IsKill);
1288 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0, Op0IsKill);
1289 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0, Op0IsKill);
1290 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0, Op0IsKill);
1291 default: return 0;
1292}
1293}
1294
1295unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
1296 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1297 return fastEmitInst_r(ARM::MVE_VREV32_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
1298 }
1299 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1300 return fastEmitInst_r(ARM::VREV32q8, &ARM::QPRRegClass, Op0, Op0IsKill);
1301 }
1302 return 0;
1303}
1304
1305unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
1306 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1307 return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
1308 }
1309 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1310 return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1311 }
1312 return 0;
1313}
1314
1315unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
1316 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1317 return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0, Op0IsKill);
1318 }
1319 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1320 return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0, Op0IsKill);
1321 }
1322 return 0;
1323}
1324
1325unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
1326 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1327 return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
1328 }
1329 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1330 return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1331 }
1332 return 0;
1333}
1334
1335unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(unsigned Op0, bool Op0IsKill) {
1336 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1337 return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1338 }
1339 return 0;
1340}
1341
1342unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
1343 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1344 return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0, Op0IsKill);
1345 }
1346 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1347 return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0, Op0IsKill);
1348 }
1349 return 0;
1350}
1351
1352unsigned fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1353switch (RetVT.SimpleTy) {
1354 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
1355 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
1356 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0, Op0IsKill);
1357 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0, Op0IsKill);
1358 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0, Op0IsKill);
1359 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0, Op0IsKill);
1360 default: return 0;
1361}
1362}
1363
1364unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
1365 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1366 return fastEmitInst_r(ARM::VREV64d8, &ARM::DPRRegClass, Op0, Op0IsKill);
1367 }
1368 return 0;
1369}
1370
1371unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
1372 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1373 return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1374 }
1375 return 0;
1376}
1377
1378unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
1379 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1380 return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0, Op0IsKill);
1381 }
1382 return 0;
1383}
1384
1385unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
1386 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1387 return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1388 }
1389 return 0;
1390}
1391
1392unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(unsigned Op0, bool Op0IsKill) {
1393 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1394 return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1395 }
1396 return 0;
1397}
1398
1399unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
1400 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1401 return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0, Op0IsKill);
1402 }
1403 return 0;
1404}
1405
1406unsigned fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1407switch (RetVT.SimpleTy) {
1408 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0, Op0IsKill);
1409 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0, Op0IsKill);
1410 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0, Op0IsKill);
1411 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0, Op0IsKill);
1412 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0, Op0IsKill);
1413 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0, Op0IsKill);
1414 default: return 0;
1415}
1416}
1417
1418unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
1419 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1420 return fastEmitInst_r(ARM::MVE_VREV64_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
1421 }
1422 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1423 return fastEmitInst_r(ARM::VREV64q8, &ARM::QPRRegClass, Op0, Op0IsKill);
1424 }
1425 return 0;
1426}
1427
1428unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
1429 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1430 return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
1431 }
1432 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1433 return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1434 }
1435 return 0;
1436}
1437
1438unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1439 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1440 return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0, Op0IsKill);
1441 }
1442 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1443 return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0, Op0IsKill);
1444 }
1445 return 0;
1446}
1447
1448unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
1449 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1450 return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
1451 }
1452 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1453 return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1454 }
1455 return 0;
1456}
1457
1458unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(unsigned Op0, bool Op0IsKill) {
1459 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1460 return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1461 }
1462 return 0;
1463}
1464
1465unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
1466 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1467 return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0, Op0IsKill);
1468 }
1469 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1470 return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0, Op0IsKill);
1471 }
1472 return 0;
1473}
1474
1475unsigned fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1476switch (RetVT.SimpleTy) {
1477 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
1478 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
1479 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
1480 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0, Op0IsKill);
1481 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0, Op0IsKill);
1482 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0, Op0IsKill);
1483 default: return 0;
1484}
1485}
1486
1487unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
1488 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1489 return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1490 }
1491 return 0;
1492}
1493
1494unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
1495 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1496 return fastEmitInst_r(ARM::VREV16d8, &ARM::DPRRegClass, Op0, Op0IsKill);
1497 }
1498 return 0;
1499}
1500
1501unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
1502 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1503 return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1504 }
1505 return 0;
1506}
1507
1508unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
1509 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1510 return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1511 }
1512 return 0;
1513}
1514
1515unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
1516 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1517 return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1518 }
1519 return 0;
1520}
1521
1522unsigned fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1523switch (RetVT.SimpleTy) {
1524 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0, Op0IsKill);
1525 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0, Op0IsKill);
1526 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0, Op0IsKill);
1527 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0, Op0IsKill);
1528 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0, Op0IsKill);
1529 default: return 0;
1530}
1531}
1532
1533unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
1534 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1535 return fastEmitInst_r(ARM::MVE_VREV16_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
1536 }
1537 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1538 return fastEmitInst_r(ARM::VREV16q8, &ARM::QPRRegClass, Op0, Op0IsKill);
1539 }
1540 return 0;
1541}
1542
1543unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1544 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1545 return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
1546 }
1547 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1548 return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1549 }
1550 return 0;
1551}
1552
1553unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
1554 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1555 return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
1556 }
1557 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1558 return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1559 }
1560 return 0;
1561}
1562
1563unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
1564 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1565 return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
1566 }
1567 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1568 return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1569 }
1570 return 0;
1571}
1572
1573unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
1574 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1575 return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
1576 }
1577 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1578 return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1579 }
1580 return 0;
1581}
1582
1583unsigned fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1584switch (RetVT.SimpleTy) {
1585 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0, Op0IsKill);
1586 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0, Op0IsKill);
1587 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0, Op0IsKill);
1588 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0, Op0IsKill);
1589 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0, Op0IsKill);
1590 default: return 0;
1591}
1592}
1593
1594unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
1595 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1596 return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1597 }
1598 return 0;
1599}
1600
1601unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
1602 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1603 return fastEmitInst_r(ARM::VREV16d8, &ARM::DPRRegClass, Op0, Op0IsKill);
1604 }
1605 return 0;
1606}
1607
1608unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
1609 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1610 return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1611 }
1612 return 0;
1613}
1614
1615unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
1616 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1617 return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1618 }
1619 return 0;
1620}
1621
1622unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
1623 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1624 return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1625 }
1626 return 0;
1627}
1628
1629unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1630switch (RetVT.SimpleTy) {
1631 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0, Op0IsKill);
1632 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0, Op0IsKill);
1633 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0, Op0IsKill);
1634 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0, Op0IsKill);
1635 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0, Op0IsKill);
1636 default: return 0;
1637}
1638}
1639
1640unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
1641 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1642 return fastEmitInst_r(ARM::VREV16q8, &ARM::QPRRegClass, Op0, Op0IsKill);
1643 }
1644 return 0;
1645}
1646
1647unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1648 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1649 return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1650 }
1651 return 0;
1652}
1653
1654unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
1655 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1656 return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1657 }
1658 return 0;
1659}
1660
1661unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
1662 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1663 return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1664 }
1665 return 0;
1666}
1667
1668unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
1669 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1670 return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1671 }
1672 return 0;
1673}
1674
1675unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1676switch (RetVT.SimpleTy) {
1677 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0, Op0IsKill);
1678 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0, Op0IsKill);
1679 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0, Op0IsKill);
1680 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0, Op0IsKill);
1681 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0, Op0IsKill);
1682 default: return 0;
1683}
1684}
1685
1686unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
1687 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1688 return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0, Op0IsKill);
1689 }
1690 return 0;
1691}
1692
1693unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
1694 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1695 return fastEmitInst_r(ARM::VREV32d8, &ARM::DPRRegClass, Op0, Op0IsKill);
1696 }
1697 return 0;
1698}
1699
1700unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
1701 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1702 return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1703 }
1704 return 0;
1705}
1706
1707unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
1708 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1709 return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0, Op0IsKill);
1710 }
1711 return 0;
1712}
1713
1714unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
1715 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1716 return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1717 }
1718 return 0;
1719}
1720
1721unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(unsigned Op0, bool Op0IsKill) {
1722 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1723 return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0, Op0IsKill);
1724 }
1725 return 0;
1726}
1727
1728unsigned fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1729switch (RetVT.SimpleTy) {
1730 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0, Op0IsKill);
1731 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0, Op0IsKill);
1732 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0, Op0IsKill);
1733 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0, Op0IsKill);
1734 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0, Op0IsKill);
1735 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0, Op0IsKill);
1736 default: return 0;
1737}
1738}
1739
1740unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
1741 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1742 return fastEmitInst_r(ARM::MVE_VREV32_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
1743 }
1744 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1745 return fastEmitInst_r(ARM::VREV32q8, &ARM::QPRRegClass, Op0, Op0IsKill);
1746 }
1747 return 0;
1748}
1749
1750unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
1751 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1752 return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
1753 }
1754 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1755 return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1756 }
1757 return 0;
1758}
1759
1760unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
1761 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1762 return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0, Op0IsKill);
1763 }
1764 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1765 return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0, Op0IsKill);
1766 }
1767 return 0;
1768}
1769
1770unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
1771 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1772 return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
1773 }
1774 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1775 return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1776 }
1777 return 0;
1778}
1779
1780unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(unsigned Op0, bool Op0IsKill) {
1781 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1782 return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1783 }
1784 return 0;
1785}
1786
1787unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
1788 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1789 return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0, Op0IsKill);
1790 }
1791 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1792 return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0, Op0IsKill);
1793 }
1794 return 0;
1795}
1796
1797unsigned fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1798switch (RetVT.SimpleTy) {
1799 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0, Op0IsKill);
1800 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0, Op0IsKill);
1801 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
1802 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0, Op0IsKill);
1803 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0, Op0IsKill);
1804 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0, Op0IsKill);
1805 default: return 0;
1806}
1807}
1808
1809unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
1810 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1811 return fastEmitInst_r(ARM::MVE_VREV64_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
1812 }
1813 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1814 return fastEmitInst_r(ARM::VREV64q8, &ARM::QPRRegClass, Op0, Op0IsKill);
1815 }
1816 return 0;
1817}
1818
1819unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
1820 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1821 return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
1822 }
1823 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1824 return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1825 }
1826 return 0;
1827}
1828
1829unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1830 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1831 return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0, Op0IsKill);
1832 }
1833 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1834 return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0, Op0IsKill);
1835 }
1836 return 0;
1837}
1838
1839unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
1840 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1841 return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0, Op0IsKill);
1842 }
1843 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1844 return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1845 }
1846 return 0;
1847}
1848
1849unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(unsigned Op0, bool Op0IsKill) {
1850 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1851 return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0, Op0IsKill);
1852 }
1853 return 0;
1854}
1855
1856unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
1857 if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
1858 return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0, Op0IsKill);
1859 }
1860 if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
1861 return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0, Op0IsKill);
1862 }
1863 return 0;
1864}
1865
1866unsigned fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1867switch (RetVT.SimpleTy) {
1868 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0, Op0IsKill);
1869 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0, Op0IsKill);
1870 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
1871 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0, Op0IsKill);
1872 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0, Op0IsKill);
1873 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0, Op0IsKill);
1874 default: return 0;
1875}
1876}
1877
1878unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1879 switch (VT.SimpleTy) {
1880 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0, Op0IsKill);
1881 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0, Op0IsKill);
1882 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0, Op0IsKill);
1883 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
1884 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
1885 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
1886 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
1887 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
1888 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
1889 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
1890 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
1891 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
1892 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
1893 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0, Op0IsKill);
1894 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0, Op0IsKill);
1895 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
1896 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
1897 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
1898 default: return 0;
1899 }
1900}
1901
1902// FastEmit functions for ISD::BITREVERSE.
1903
1904unsigned fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1905 if (RetVT.SimpleTy != MVT::i32)
1906 return 0;
1907 if ((Subtarget->isThumb2())) {
1908 return fastEmitInst_r(ARM::t2RBIT, &ARM::rGPRRegClass, Op0, Op0IsKill);
1909 }
1910 if ((Subtarget->hasV6T2Ops()) && (!Subtarget->isThumb())) {
1911 return fastEmitInst_r(ARM::RBIT, &ARM::GPRRegClass, Op0, Op0IsKill);
1912 }
1913 return 0;
1914}
1915
1916unsigned fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1917 switch (VT.SimpleTy) {
1918 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0, Op0IsKill);
1919 default: return 0;
1920 }
1921}
1922
1923// FastEmit functions for ISD::BRIND.
1924
1925unsigned fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1926 if (RetVT.SimpleTy != MVT::isVoid)
1927 return 0;
1928 if ((Subtarget->isThumb())) {
1929 return fastEmitInst_r(ARM::tBRIND, &ARM::GPRRegClass, Op0, Op0IsKill);
1930 }
1931 if ((!Subtarget->isThumb()) && (!Subtarget->hasV4TOps())) {
1932 return fastEmitInst_r(ARM::MOVPCRX, &ARM::GPRRegClass, Op0, Op0IsKill);
1933 }
1934 if ((Subtarget->hasV4TOps()) && (!Subtarget->isThumb())) {
1935 return fastEmitInst_r(ARM::BX, &ARM::GPRRegClass, Op0, Op0IsKill);
1936 }
1937 return 0;
1938}
1939
1940unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1941 switch (VT.SimpleTy) {
1942 case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill);
1943 default: return 0;
1944 }
1945}
1946
1947// FastEmit functions for ISD::BSWAP.
1948
1949unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1950 if (RetVT.SimpleTy != MVT::i32)
1951 return 0;
1952 if ((Subtarget->isThumb2())) {
1953 return fastEmitInst_r(ARM::t2REV, &ARM::rGPRRegClass, Op0, Op0IsKill);
1954 }
1955 if ((Subtarget->hasV6Ops()) && (Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
1956 return fastEmitInst_r(ARM::tREV, &ARM::tGPRRegClass, Op0, Op0IsKill);
1957 }
1958 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
1959 return fastEmitInst_r(ARM::REV, &ARM::GPRRegClass, Op0, Op0IsKill);
1960 }
1961 return 0;
1962}
1963
1964unsigned fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1965 if (RetVT.SimpleTy != MVT::v8i16)
1966 return 0;
1967 if ((Subtarget->hasMVEIntegerOps())) {
1968 return fastEmitInst_r(ARM::MVE_VREV16_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
1969 }
1970 return 0;
1971}
1972
1973unsigned fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1974 if (RetVT.SimpleTy != MVT::v4i32)
1975 return 0;
1976 if ((Subtarget->hasMVEIntegerOps())) {
1977 return fastEmitInst_r(ARM::MVE_VREV32_8, &ARM::MQPRRegClass, Op0, Op0IsKill);
1978 }
1979 return 0;
1980}
1981
1982unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1983 switch (VT.SimpleTy) {
1984 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0, Op0IsKill);
1985 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
1986 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
1987 default: return 0;
1988 }
1989}
1990
1991// FastEmit functions for ISD::CTLZ.
1992
1993unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1994 if (RetVT.SimpleTy != MVT::i32)
1995 return 0;
1996 if ((Subtarget->isThumb2())) {
1997 return fastEmitInst_r(ARM::t2CLZ, &ARM::rGPRRegClass, Op0, Op0IsKill);
1998 }
1999 if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb())) {
2000 return fastEmitInst_r(ARM::CLZ, &ARM::GPRRegClass, Op0, Op0IsKill);
2001 }
2002 return 0;
2003}
2004
2005unsigned fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2006 if (RetVT.SimpleTy != MVT::v8i8)
2007 return 0;
2008 if ((Subtarget->hasNEON())) {
2009 return fastEmitInst_r(ARM::VCLZv8i8, &ARM::DPRRegClass, Op0, Op0IsKill);
2010 }
2011 return 0;
2012}
2013
2014unsigned fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2015 if (RetVT.SimpleTy != MVT::v16i8)
2016 return 0;
2017 if ((Subtarget->hasMVEIntegerOps())) {
2018 return fastEmitInst_r(ARM::MVE_VCLZs8, &ARM::MQPRRegClass, Op0, Op0IsKill);
2019 }
2020 if ((Subtarget->hasNEON())) {
2021 return fastEmitInst_r(ARM::VCLZv16i8, &ARM::QPRRegClass, Op0, Op0IsKill);
2022 }
2023 return 0;
2024}
2025
2026unsigned fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2027 if (RetVT.SimpleTy != MVT::v4i16)
2028 return 0;
2029 if ((Subtarget->hasNEON())) {
2030 return fastEmitInst_r(ARM::VCLZv4i16, &ARM::DPRRegClass, Op0, Op0IsKill);
2031 }
2032 return 0;
2033}
2034
2035unsigned fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2036 if (RetVT.SimpleTy != MVT::v8i16)
2037 return 0;
2038 if ((Subtarget->hasMVEIntegerOps())) {
2039 return fastEmitInst_r(ARM::MVE_VCLZs16, &ARM::MQPRRegClass, Op0, Op0IsKill);
2040 }
2041 if ((Subtarget->hasNEON())) {
2042 return fastEmitInst_r(ARM::VCLZv8i16, &ARM::QPRRegClass, Op0, Op0IsKill);
2043 }
2044 return 0;
2045}
2046
2047unsigned fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2048 if (RetVT.SimpleTy != MVT::v2i32)
2049 return 0;
2050 if ((Subtarget->hasNEON())) {
2051 return fastEmitInst_r(ARM::VCLZv2i32, &ARM::DPRRegClass, Op0, Op0IsKill);
2052 }
2053 return 0;
2054}
2055
2056unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2057 if (RetVT.SimpleTy != MVT::v4i32)
2058 return 0;
2059 if ((Subtarget->hasMVEIntegerOps())) {
2060 return fastEmitInst_r(ARM::MVE_VCLZs32, &ARM::MQPRRegClass, Op0, Op0IsKill);
2061 }
2062 if ((Subtarget->hasNEON())) {
2063 return fastEmitInst_r(ARM::VCLZv4i32, &ARM::QPRRegClass, Op0, Op0IsKill);
2064 }
2065 return 0;
2066}
2067
2068unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2069 switch (VT.SimpleTy) {
2070 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
2071 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
2072 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
2073 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
2074 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
2075 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
2076 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2077 default: return 0;
2078 }
2079}
2080
2081// FastEmit functions for ISD::CTPOP.
2082
2083unsigned fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2084 if (RetVT.SimpleTy != MVT::v8i8)
2085 return 0;
2086 if ((Subtarget->hasNEON())) {
2087 return fastEmitInst_r(ARM::VCNTd, &ARM::DPRRegClass, Op0, Op0IsKill);
2088 }
2089 return 0;
2090}
2091
2092unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2093 if (RetVT.SimpleTy != MVT::v16i8)
2094 return 0;
2095 if ((Subtarget->hasNEON())) {
2096 return fastEmitInst_r(ARM::VCNTq, &ARM::QPRRegClass, Op0, Op0IsKill);
2097 }
2098 return 0;
2099}
2100
2101unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2102 switch (VT.SimpleTy) {
2103 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
2104 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
2105 default: return 0;
2106 }
2107}
2108
2109// FastEmit functions for ISD::FABS.
2110
2111unsigned fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2112 if (RetVT.SimpleTy != MVT::f16)
2113 return 0;
2114 if ((Subtarget->hasFullFP16())) {
2115 return fastEmitInst_r(ARM::VABSH, &ARM::HPRRegClass, Op0, Op0IsKill);
2116 }
2117 return 0;
2118}
2119
2120unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2121 if (RetVT.SimpleTy != MVT::f32)
2122 return 0;
2123 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
2124 return fastEmitInst_r(ARM::VABSS, &ARM::SPRRegClass, Op0, Op0IsKill);
2125 }
2126 return 0;
2127}
2128
2129unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2130 if (RetVT.SimpleTy != MVT::f64)
2131 return 0;
2132 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2133 return fastEmitInst_r(ARM::VABSD, &ARM::DPRRegClass, Op0, Op0IsKill);
2134 }
2135 return 0;
2136}
2137
2138unsigned fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2139 if (RetVT.SimpleTy != MVT::v4f16)
2140 return 0;
2141 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2142 return fastEmitInst_r(ARM::VABShd, &ARM::DPRRegClass, Op0, Op0IsKill);
2143 }
2144 return 0;
2145}
2146
2147unsigned fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2148 if (RetVT.SimpleTy != MVT::v8f16)
2149 return 0;
2150 if ((Subtarget->hasMVEIntegerOps())) {
2151 return fastEmitInst_r(ARM::MVE_VABSf16, &ARM::MQPRRegClass, Op0, Op0IsKill);
2152 }
2153 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2154 return fastEmitInst_r(ARM::VABShq, &ARM::QPRRegClass, Op0, Op0IsKill);
2155 }
2156 return 0;
2157}
2158
2159unsigned fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2160 if (RetVT.SimpleTy != MVT::v2f32)
2161 return 0;
2162 if ((Subtarget->hasNEON())) {
2163 return fastEmitInst_r(ARM::VABSfd, &ARM::DPRRegClass, Op0, Op0IsKill);
2164 }
2165 return 0;
2166}
2167
2168unsigned fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2169 if (RetVT.SimpleTy != MVT::v4f32)
2170 return 0;
2171 if ((Subtarget->hasMVEIntegerOps())) {
2172 return fastEmitInst_r(ARM::MVE_VABSf32, &ARM::MQPRRegClass, Op0, Op0IsKill);
2173 }
2174 if ((Subtarget->hasNEON())) {
2175 return fastEmitInst_r(ARM::VABSfq, &ARM::QPRRegClass, Op0, Op0IsKill);
2176 }
2177 return 0;
2178}
2179
2180unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2181 switch (VT.SimpleTy) {
2182 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0, Op0IsKill);
2183 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0, Op0IsKill);
2184 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0, Op0IsKill);
2185 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
2186 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
2187 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
2188 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2189 default: return 0;
2190 }
2191}
2192
2193// FastEmit functions for ISD::FCEIL.
2194
2195unsigned fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2196 if (RetVT.SimpleTy != MVT::f16)
2197 return 0;
2198 if ((Subtarget->hasFullFP16())) {
2199 return fastEmitInst_r(ARM::VRINTPH, &ARM::HPRRegClass, Op0, Op0IsKill);
2200 }
2201 return 0;
2202}
2203
2204unsigned fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2205 if (RetVT.SimpleTy != MVT::f32)
2206 return 0;
2207 if ((Subtarget->hasFPARMv8Base())) {
2208 return fastEmitInst_r(ARM::VRINTPS, &ARM::SPRRegClass, Op0, Op0IsKill);
2209 }
2210 return 0;
2211}
2212
2213unsigned fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2214 if (RetVT.SimpleTy != MVT::f64)
2215 return 0;
2216 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2217 return fastEmitInst_r(ARM::VRINTPD, &ARM::DPRRegClass, Op0, Op0IsKill);
2218 }
2219 return 0;
2220}
2221
2222unsigned fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2223 if (RetVT.SimpleTy != MVT::v8f16)
2224 return 0;
2225 if ((Subtarget->hasMVEFloatOps())) {
2226 return fastEmitInst_r(ARM::MVE_VRINTf16P, &ARM::MQPRRegClass, Op0, Op0IsKill);
2227 }
2228 return 0;
2229}
2230
2231unsigned fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2232 if (RetVT.SimpleTy != MVT::v4f32)
2233 return 0;
2234 if ((Subtarget->hasMVEFloatOps())) {
2235 return fastEmitInst_r(ARM::MVE_VRINTf32P, &ARM::MQPRRegClass, Op0, Op0IsKill);
2236 }
2237 return 0;
2238}
2239
2240unsigned fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2241 switch (VT.SimpleTy) {
2242 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0, Op0IsKill);
2243 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0, Op0IsKill);
2244 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0, Op0IsKill);
2245 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
2246 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2247 default: return 0;
2248 }
2249}
2250
2251// FastEmit functions for ISD::FFLOOR.
2252
2253unsigned fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2254 if (RetVT.SimpleTy != MVT::f16)
2255 return 0;
2256 if ((Subtarget->hasFullFP16())) {
2257 return fastEmitInst_r(ARM::VRINTMH, &ARM::HPRRegClass, Op0, Op0IsKill);
2258 }
2259 return 0;
2260}
2261
2262unsigned fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2263 if (RetVT.SimpleTy != MVT::f32)
2264 return 0;
2265 if ((Subtarget->hasFPARMv8Base())) {
2266 return fastEmitInst_r(ARM::VRINTMS, &ARM::SPRRegClass, Op0, Op0IsKill);
2267 }
2268 return 0;
2269}
2270
2271unsigned fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2272 if (RetVT.SimpleTy != MVT::f64)
2273 return 0;
2274 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2275 return fastEmitInst_r(ARM::VRINTMD, &ARM::DPRRegClass, Op0, Op0IsKill);
2276 }
2277 return 0;
2278}
2279
2280unsigned fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2281 if (RetVT.SimpleTy != MVT::v8f16)
2282 return 0;
2283 if ((Subtarget->hasMVEFloatOps())) {
2284 return fastEmitInst_r(ARM::MVE_VRINTf16M, &ARM::MQPRRegClass, Op0, Op0IsKill);
2285 }
2286 return 0;
2287}
2288
2289unsigned fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2290 if (RetVT.SimpleTy != MVT::v4f32)
2291 return 0;
2292 if ((Subtarget->hasMVEFloatOps())) {
2293 return fastEmitInst_r(ARM::MVE_VRINTf32M, &ARM::MQPRRegClass, Op0, Op0IsKill);
2294 }
2295 return 0;
2296}
2297
2298unsigned fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2299 switch (VT.SimpleTy) {
2300 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0, Op0IsKill);
2301 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0, Op0IsKill);
2302 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0, Op0IsKill);
2303 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
2304 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2305 default: return 0;
2306 }
2307}
2308
2309// FastEmit functions for ISD::FNEARBYINT.
2310
2311unsigned fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2312 if (RetVT.SimpleTy != MVT::f16)
2313 return 0;
2314 if ((Subtarget->hasFullFP16())) {
2315 return fastEmitInst_r(ARM::VRINTRH, &ARM::HPRRegClass, Op0, Op0IsKill);
2316 }
2317 return 0;
2318}
2319
2320unsigned fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2321 if (RetVT.SimpleTy != MVT::f32)
2322 return 0;
2323 if ((Subtarget->hasFPARMv8Base())) {
2324 return fastEmitInst_r(ARM::VRINTRS, &ARM::SPRRegClass, Op0, Op0IsKill);
2325 }
2326 return 0;
2327}
2328
2329unsigned fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2330 if (RetVT.SimpleTy != MVT::f64)
2331 return 0;
2332 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2333 return fastEmitInst_r(ARM::VRINTRD, &ARM::DPRRegClass, Op0, Op0IsKill);
2334 }
2335 return 0;
2336}
2337
2338unsigned fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2339 switch (VT.SimpleTy) {
2340 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0, Op0IsKill);
2341 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
2342 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
2343 default: return 0;
2344 }
2345}
2346
2347// FastEmit functions for ISD::FNEG.
2348
2349unsigned fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2350 if (RetVT.SimpleTy != MVT::f16)
2351 return 0;
2352 if ((Subtarget->hasFullFP16())) {
2353 return fastEmitInst_r(ARM::VNEGH, &ARM::HPRRegClass, Op0, Op0IsKill);
2354 }
2355 return 0;
2356}
2357
2358unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2359 if (RetVT.SimpleTy != MVT::f32)
2360 return 0;
2361 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
2362 return fastEmitInst_r(ARM::VNEGS, &ARM::SPRRegClass, Op0, Op0IsKill);
2363 }
2364 return 0;
2365}
2366
2367unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2368 if (RetVT.SimpleTy != MVT::f64)
2369 return 0;
2370 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2371 return fastEmitInst_r(ARM::VNEGD, &ARM::DPRRegClass, Op0, Op0IsKill);
2372 }
2373 return 0;
2374}
2375
2376unsigned fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2377 if (RetVT.SimpleTy != MVT::v4f16)
2378 return 0;
2379 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2380 return fastEmitInst_r(ARM::VNEGhd, &ARM::DPRRegClass, Op0, Op0IsKill);
2381 }
2382 return 0;
2383}
2384
2385unsigned fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2386 if (RetVT.SimpleTy != MVT::v8f16)
2387 return 0;
2388 if ((Subtarget->hasMVEIntegerOps())) {
2389 return fastEmitInst_r(ARM::MVE_VNEGf16, &ARM::MQPRRegClass, Op0, Op0IsKill);
2390 }
2391 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2392 return fastEmitInst_r(ARM::VNEGhq, &ARM::QPRRegClass, Op0, Op0IsKill);
2393 }
2394 return 0;
2395}
2396
2397unsigned fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2398 if (RetVT.SimpleTy != MVT::v2f32)
2399 return 0;
2400 if ((Subtarget->hasNEON())) {
2401 return fastEmitInst_r(ARM::VNEGfd, &ARM::DPRRegClass, Op0, Op0IsKill);
2402 }
2403 return 0;
2404}
2405
2406unsigned fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2407 if (RetVT.SimpleTy != MVT::v4f32)
2408 return 0;
2409 if ((Subtarget->hasMVEIntegerOps())) {
2410 return fastEmitInst_r(ARM::MVE_VNEGf32, &ARM::MQPRRegClass, Op0, Op0IsKill);
2411 }
2412 if ((Subtarget->hasNEON())) {
2413 return fastEmitInst_r(ARM::VNEGf32q, &ARM::QPRRegClass, Op0, Op0IsKill);
2414 }
2415 return 0;
2416}
2417
2418unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2419 switch (VT.SimpleTy) {
2420 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0, Op0IsKill);
2421 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0, Op0IsKill);
2422 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0, Op0IsKill);
2423 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
2424 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
2425 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
2426 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2427 default: return 0;
2428 }
2429}
2430
2431// FastEmit functions for ISD::FP_EXTEND.
2432
2433unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2434 if (RetVT.SimpleTy != MVT::f64)
2435 return 0;
2436 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2437 return fastEmitInst_r(ARM::VCVTDS, &ARM::DPRRegClass, Op0, Op0IsKill);
2438 }
2439 return 0;
2440}
2441
2442unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2443 switch (VT.SimpleTy) {
2444 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0, Op0IsKill);
2445 default: return 0;
2446 }
2447}
2448
2449// FastEmit functions for ISD::FP_ROUND.
2450
2451unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2452 if (RetVT.SimpleTy != MVT::f32)
2453 return 0;
2454 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2455 return fastEmitInst_r(ARM::VCVTSD, &ARM::SPRRegClass, Op0, Op0IsKill);
2456 }
2457 return 0;
2458}
2459
2460unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2461 switch (VT.SimpleTy) {
2462 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
2463 default: return 0;
2464 }
2465}
2466
2467// FastEmit functions for ISD::FP_TO_SINT.
2468
2469unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2470 if (RetVT.SimpleTy != MVT::v4i16)
2471 return 0;
2472 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2473 return fastEmitInst_r(ARM::VCVTh2sd, &ARM::DPRRegClass, Op0, Op0IsKill);
2474 }
2475 return 0;
2476}
2477
2478unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2479 if (RetVT.SimpleTy != MVT::v8i16)
2480 return 0;
2481 if ((Subtarget->hasMVEFloatOps())) {
2482 return fastEmitInst_r(ARM::MVE_VCVTs16f16z, &ARM::MQPRRegClass, Op0, Op0IsKill);
2483 }
2484 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2485 return fastEmitInst_r(ARM::VCVTh2sq, &ARM::QPRRegClass, Op0, Op0IsKill);
2486 }
2487 return 0;
2488}
2489
2490unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2491 if (RetVT.SimpleTy != MVT::v2i32)
2492 return 0;
2493 if ((Subtarget->hasNEON())) {
2494 return fastEmitInst_r(ARM::VCVTf2sd, &ARM::DPRRegClass, Op0, Op0IsKill);
2495 }
2496 return 0;
2497}
2498
2499unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2500 if (RetVT.SimpleTy != MVT::v4i32)
2501 return 0;
2502 if ((Subtarget->hasMVEFloatOps())) {
2503 return fastEmitInst_r(ARM::MVE_VCVTs32f32z, &ARM::MQPRRegClass, Op0, Op0IsKill);
2504 }
2505 if ((Subtarget->hasNEON())) {
2506 return fastEmitInst_r(ARM::VCVTf2sq, &ARM::QPRRegClass, Op0, Op0IsKill);
2507 }
2508 return 0;
2509}
2510
2511unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2512 switch (VT.SimpleTy) {
2513 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
2514 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
2515 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
2516 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2517 default: return 0;
2518 }
2519}
2520
2521// FastEmit functions for ISD::FP_TO_UINT.
2522
2523unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2524 if (RetVT.SimpleTy != MVT::v4i16)
2525 return 0;
2526 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2527 return fastEmitInst_r(ARM::VCVTh2ud, &ARM::DPRRegClass, Op0, Op0IsKill);
2528 }
2529 return 0;
2530}
2531
2532unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2533 if (RetVT.SimpleTy != MVT::v8i16)
2534 return 0;
2535 if ((Subtarget->hasMVEFloatOps())) {
2536 return fastEmitInst_r(ARM::MVE_VCVTu16f16z, &ARM::MQPRRegClass, Op0, Op0IsKill);
2537 }
2538 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2539 return fastEmitInst_r(ARM::VCVTh2uq, &ARM::QPRRegClass, Op0, Op0IsKill);
2540 }
2541 return 0;
2542}
2543
2544unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2545 if (RetVT.SimpleTy != MVT::v2i32)
2546 return 0;
2547 if ((Subtarget->hasNEON())) {
2548 return fastEmitInst_r(ARM::VCVTf2ud, &ARM::DPRRegClass, Op0, Op0IsKill);
2549 }
2550 return 0;
2551}
2552
2553unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2554 if (RetVT.SimpleTy != MVT::v4i32)
2555 return 0;
2556 if ((Subtarget->hasMVEFloatOps())) {
2557 return fastEmitInst_r(ARM::MVE_VCVTu32f32z, &ARM::MQPRRegClass, Op0, Op0IsKill);
2558 }
2559 if ((Subtarget->hasNEON())) {
2560 return fastEmitInst_r(ARM::VCVTf2uq, &ARM::QPRRegClass, Op0, Op0IsKill);
2561 }
2562 return 0;
2563}
2564
2565unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2566 switch (VT.SimpleTy) {
2567 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
2568 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
2569 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
2570 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2571 default: return 0;
2572 }
2573}
2574
2575// FastEmit functions for ISD::FRINT.
2576
2577unsigned fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2578 if (RetVT.SimpleTy != MVT::f16)
2579 return 0;
2580 if ((Subtarget->hasFullFP16())) {
2581 return fastEmitInst_r(ARM::VRINTXH, &ARM::HPRRegClass, Op0, Op0IsKill);
2582 }
2583 return 0;
2584}
2585
2586unsigned fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2587 if (RetVT.SimpleTy != MVT::f32)
2588 return 0;
2589 if ((Subtarget->hasFPARMv8Base())) {
2590 return fastEmitInst_r(ARM::VRINTXS, &ARM::SPRRegClass, Op0, Op0IsKill);
2591 }
2592 return 0;
2593}
2594
2595unsigned fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2596 if (RetVT.SimpleTy != MVT::f64)
2597 return 0;
2598 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2599 return fastEmitInst_r(ARM::VRINTXD, &ARM::DPRRegClass, Op0, Op0IsKill);
2600 }
2601 return 0;
2602}
2603
2604unsigned fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2605 if (RetVT.SimpleTy != MVT::v8f16)
2606 return 0;
2607 if ((Subtarget->hasMVEFloatOps())) {
2608 return fastEmitInst_r(ARM::MVE_VRINTf16X, &ARM::MQPRRegClass, Op0, Op0IsKill);
2609 }
2610 return 0;
2611}
2612
2613unsigned fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2614 if (RetVT.SimpleTy != MVT::v4f32)
2615 return 0;
2616 if ((Subtarget->hasMVEFloatOps())) {
2617 return fastEmitInst_r(ARM::MVE_VRINTf32X, &ARM::MQPRRegClass, Op0, Op0IsKill);
2618 }
2619 return 0;
2620}
2621
2622unsigned fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2623 switch (VT.SimpleTy) {
2624 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0, Op0IsKill);
2625 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
2626 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
2627 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
2628 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2629 default: return 0;
2630 }
2631}
2632
2633// FastEmit functions for ISD::FROUND.
2634
2635unsigned fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2636 if (RetVT.SimpleTy != MVT::f16)
2637 return 0;
2638 if ((Subtarget->hasFullFP16())) {
2639 return fastEmitInst_r(ARM::VRINTAH, &ARM::HPRRegClass, Op0, Op0IsKill);
2640 }
2641 return 0;
2642}
2643
2644unsigned fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2645 if (RetVT.SimpleTy != MVT::f32)
2646 return 0;
2647 if ((Subtarget->hasFPARMv8Base())) {
2648 return fastEmitInst_r(ARM::VRINTAS, &ARM::SPRRegClass, Op0, Op0IsKill);
2649 }
2650 return 0;
2651}
2652
2653unsigned fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2654 if (RetVT.SimpleTy != MVT::f64)
2655 return 0;
2656 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2657 return fastEmitInst_r(ARM::VRINTAD, &ARM::DPRRegClass, Op0, Op0IsKill);
2658 }
2659 return 0;
2660}
2661
2662unsigned fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2663 if (RetVT.SimpleTy != MVT::v8f16)
2664 return 0;
2665 if ((Subtarget->hasMVEFloatOps())) {
2666 return fastEmitInst_r(ARM::MVE_VRINTf16A, &ARM::MQPRRegClass, Op0, Op0IsKill);
2667 }
2668 return 0;
2669}
2670
2671unsigned fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2672 if (RetVT.SimpleTy != MVT::v4f32)
2673 return 0;
2674 if ((Subtarget->hasMVEFloatOps())) {
2675 return fastEmitInst_r(ARM::MVE_VRINTf32A, &ARM::MQPRRegClass, Op0, Op0IsKill);
2676 }
2677 return 0;
2678}
2679
2680unsigned fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2681 switch (VT.SimpleTy) {
2682 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0, Op0IsKill);
2683 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0, Op0IsKill);
2684 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
2685 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
2686 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2687 default: return 0;
2688 }
2689}
2690
2691// FastEmit functions for ISD::FSQRT.
2692
2693unsigned fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2694 if (RetVT.SimpleTy != MVT::f16)
2695 return 0;
2696 if ((Subtarget->hasFullFP16())) {
2697 return fastEmitInst_r(ARM::VSQRTH, &ARM::HPRRegClass, Op0, Op0IsKill);
2698 }
2699 return 0;
2700}
2701
2702unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2703 if (RetVT.SimpleTy != MVT::f32)
2704 return 0;
2705 if ((Subtarget->hasVFP2Base())) {
2706 return fastEmitInst_r(ARM::VSQRTS, &ARM::SPRRegClass, Op0, Op0IsKill);
2707 }
2708 return 0;
2709}
2710
2711unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2712 if (RetVT.SimpleTy != MVT::f64)
2713 return 0;
2714 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
2715 return fastEmitInst_r(ARM::VSQRTD, &ARM::DPRRegClass, Op0, Op0IsKill);
2716 }
2717 return 0;
2718}
2719
2720unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2721 switch (VT.SimpleTy) {
2722 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0, Op0IsKill);
2723 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
2724 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0, Op0IsKill);
2725 default: return 0;
2726 }
2727}
2728
2729// FastEmit functions for ISD::FTRUNC.
2730
2731unsigned fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2732 if (RetVT.SimpleTy != MVT::f16)
2733 return 0;
2734 if ((Subtarget->hasFullFP16())) {
2735 return fastEmitInst_r(ARM::VRINTZH, &ARM::HPRRegClass, Op0, Op0IsKill);
2736 }
2737 return 0;
2738}
2739
2740unsigned fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2741 if (RetVT.SimpleTy != MVT::f32)
2742 return 0;
2743 if ((Subtarget->hasFPARMv8Base())) {
2744 return fastEmitInst_r(ARM::VRINTZS, &ARM::SPRRegClass, Op0, Op0IsKill);
2745 }
2746 return 0;
2747}
2748
2749unsigned fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2750 if (RetVT.SimpleTy != MVT::f64)
2751 return 0;
2752 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
2753 return fastEmitInst_r(ARM::VRINTZD, &ARM::DPRRegClass, Op0, Op0IsKill);
2754 }
2755 return 0;
2756}
2757
2758unsigned fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2759 if (RetVT.SimpleTy != MVT::v8f16)
2760 return 0;
2761 if ((Subtarget->hasMVEFloatOps())) {
2762 return fastEmitInst_r(ARM::MVE_VRINTf16Z, &ARM::MQPRRegClass, Op0, Op0IsKill);
2763 }
2764 return 0;
2765}
2766
2767unsigned fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2768 if (RetVT.SimpleTy != MVT::v4f32)
2769 return 0;
2770 if ((Subtarget->hasMVEFloatOps())) {
2771 return fastEmitInst_r(ARM::MVE_VRINTf32Z, &ARM::MQPRRegClass, Op0, Op0IsKill);
2772 }
2773 return 0;
2774}
2775
2776unsigned fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2777 switch (VT.SimpleTy) {
2778 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0, Op0IsKill);
2779 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0, Op0IsKill);
2780 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0, Op0IsKill);
2781 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
2782 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2783 default: return 0;
2784 }
2785}
2786
2787// FastEmit functions for ISD::SIGN_EXTEND.
2788
2789unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2790 if (RetVT.SimpleTy != MVT::v8i16)
2791 return 0;
2792 if ((Subtarget->hasNEON())) {
2793 return fastEmitInst_r(ARM::VMOVLsv8i16, &ARM::QPRRegClass, Op0, Op0IsKill);
2794 }
2795 return 0;
2796}
2797
2798unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2799 if (RetVT.SimpleTy != MVT::v4i32)
2800 return 0;
2801 if ((Subtarget->hasNEON())) {
2802 return fastEmitInst_r(ARM::VMOVLsv4i32, &ARM::QPRRegClass, Op0, Op0IsKill);
2803 }
2804 return 0;
2805}
2806
2807unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2808 if (RetVT.SimpleTy != MVT::v2i64)
2809 return 0;
2810 if ((Subtarget->hasNEON())) {
2811 return fastEmitInst_r(ARM::VMOVLsv2i64, &ARM::QPRRegClass, Op0, Op0IsKill);
2812 }
2813 return 0;
2814}
2815
2816unsigned fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2817 switch (VT.SimpleTy) {
2818 case MVT::v8i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
2819 case MVT::v4i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
2820 case MVT::v2i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
2821 default: return 0;
2822 }
2823}
2824
2825// FastEmit functions for ISD::SINT_TO_FP.
2826
2827unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2828 if (RetVT.SimpleTy != MVT::v4f16)
2829 return 0;
2830 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2831 return fastEmitInst_r(ARM::VCVTs2hd, &ARM::DPRRegClass, Op0, Op0IsKill);
2832 }
2833 return 0;
2834}
2835
2836unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2837 if (RetVT.SimpleTy != MVT::v8f16)
2838 return 0;
2839 if ((Subtarget->hasMVEFloatOps())) {
2840 return fastEmitInst_r(ARM::MVE_VCVTf16s16n, &ARM::MQPRRegClass, Op0, Op0IsKill);
2841 }
2842 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2843 return fastEmitInst_r(ARM::VCVTs2hq, &ARM::QPRRegClass, Op0, Op0IsKill);
2844 }
2845 return 0;
2846}
2847
2848unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2849 if (RetVT.SimpleTy != MVT::v2f32)
2850 return 0;
2851 if ((Subtarget->hasNEON())) {
2852 return fastEmitInst_r(ARM::VCVTs2fd, &ARM::DPRRegClass, Op0, Op0IsKill);
2853 }
2854 return 0;
2855}
2856
2857unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2858 if (RetVT.SimpleTy != MVT::v4f32)
2859 return 0;
2860 if ((Subtarget->hasMVEFloatOps())) {
2861 return fastEmitInst_r(ARM::MVE_VCVTf32s32n, &ARM::MQPRRegClass, Op0, Op0IsKill);
2862 }
2863 if ((Subtarget->hasNEON())) {
2864 return fastEmitInst_r(ARM::VCVTs2fq, &ARM::QPRRegClass, Op0, Op0IsKill);
2865 }
2866 return 0;
2867}
2868
2869unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2870 switch (VT.SimpleTy) {
2871 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
2872 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
2873 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
2874 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2875 default: return 0;
2876 }
2877}
2878
2879// FastEmit functions for ISD::TRUNCATE.
2880
2881unsigned fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2882 if (RetVT.SimpleTy != MVT::v8i8)
2883 return 0;
2884 if ((Subtarget->hasNEON())) {
2885 return fastEmitInst_r(ARM::VMOVNv8i8, &ARM::DPRRegClass, Op0, Op0IsKill);
2886 }
2887 return 0;
2888}
2889
2890unsigned fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2891 if (RetVT.SimpleTy != MVT::v4i16)
2892 return 0;
2893 if ((Subtarget->hasNEON())) {
2894 return fastEmitInst_r(ARM::VMOVNv4i16, &ARM::DPRRegClass, Op0, Op0IsKill);
2895 }
2896 return 0;
2897}
2898
2899unsigned fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2900 if (RetVT.SimpleTy != MVT::v2i32)
2901 return 0;
2902 if ((Subtarget->hasNEON())) {
2903 return fastEmitInst_r(ARM::VMOVNv2i32, &ARM::DPRRegClass, Op0, Op0IsKill);
2904 }
2905 return 0;
2906}
2907
2908unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2909 switch (VT.SimpleTy) {
2910 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
2911 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2912 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
2913 default: return 0;
2914 }
2915}
2916
2917// FastEmit functions for ISD::UINT_TO_FP.
2918
2919unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2920 if (RetVT.SimpleTy != MVT::v4f16)
2921 return 0;
2922 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2923 return fastEmitInst_r(ARM::VCVTu2hd, &ARM::DPRRegClass, Op0, Op0IsKill);
2924 }
2925 return 0;
2926}
2927
2928unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2929 if (RetVT.SimpleTy != MVT::v8f16)
2930 return 0;
2931 if ((Subtarget->hasMVEFloatOps())) {
2932 return fastEmitInst_r(ARM::MVE_VCVTf16u16n, &ARM::MQPRRegClass, Op0, Op0IsKill);
2933 }
2934 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
2935 return fastEmitInst_r(ARM::VCVTu2hq, &ARM::QPRRegClass, Op0, Op0IsKill);
2936 }
2937 return 0;
2938}
2939
2940unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2941 if (RetVT.SimpleTy != MVT::v2f32)
2942 return 0;
2943 if ((Subtarget->hasNEON())) {
2944 return fastEmitInst_r(ARM::VCVTu2fd, &ARM::DPRRegClass, Op0, Op0IsKill);
2945 }
2946 return 0;
2947}
2948
2949unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2950 if (RetVT.SimpleTy != MVT::v4f32)
2951 return 0;
2952 if ((Subtarget->hasMVEFloatOps())) {
2953 return fastEmitInst_r(ARM::MVE_VCVTf32u32n, &ARM::MQPRRegClass, Op0, Op0IsKill);
2954 }
2955 if ((Subtarget->hasNEON())) {
2956 return fastEmitInst_r(ARM::VCVTu2fq, &ARM::QPRRegClass, Op0, Op0IsKill);
2957 }
2958 return 0;
2959}
2960
2961unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2962 switch (VT.SimpleTy) {
2963 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
2964 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
2965 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
2966 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2967 default: return 0;
2968 }
2969}
2970
2971// FastEmit functions for ISD::VECREDUCE_ADD.
2972
2973unsigned fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2974 if (RetVT.SimpleTy != MVT::i32)
2975 return 0;
2976 if ((Subtarget->hasMVEIntegerOps())) {
2977 return fastEmitInst_r(ARM::MVE_VADDVu8no_acc, &ARM::tGPREvenRegClass, Op0, Op0IsKill);
2978 }
2979 return 0;
2980}
2981
2982unsigned fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2983 if (RetVT.SimpleTy != MVT::i32)
2984 return 0;
2985 if ((Subtarget->hasMVEIntegerOps())) {
2986 return fastEmitInst_r(ARM::MVE_VADDVu16no_acc, &ARM::tGPREvenRegClass, Op0, Op0IsKill);
2987 }
2988 return 0;
2989}
2990
2991unsigned fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2992 if (RetVT.SimpleTy != MVT::i32)
2993 return 0;
2994 if ((Subtarget->hasMVEIntegerOps())) {
2995 return fastEmitInst_r(ARM::MVE_VADDVu32no_acc, &ARM::tGPREvenRegClass, Op0, Op0IsKill);
2996 }
2997 return 0;
2998}
2999
3000unsigned fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3001 switch (VT.SimpleTy) {
3002 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
3003 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
3004 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
3005 default: return 0;
3006 }
3007}
3008
3009// FastEmit functions for ISD::ZERO_EXTEND.
3010
3011unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3012 if (RetVT.SimpleTy != MVT::v8i16)
3013 return 0;
3014 if ((Subtarget->hasNEON())) {
3015 return fastEmitInst_r(ARM::VMOVLuv8i16, &ARM::QPRRegClass, Op0, Op0IsKill);
3016 }
3017 return 0;
3018}
3019
3020unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3021 if (RetVT.SimpleTy != MVT::v4i32)
3022 return 0;
3023 if ((Subtarget->hasNEON())) {
3024 return fastEmitInst_r(ARM::VMOVLuv4i32, &ARM::QPRRegClass, Op0, Op0IsKill);
3025 }
3026 return 0;
3027}
3028
3029unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3030 if (RetVT.SimpleTy != MVT::v2i64)
3031 return 0;
3032 if ((Subtarget->hasNEON())) {
3033 return fastEmitInst_r(ARM::VMOVLuv2i64, &ARM::QPRRegClass, Op0, Op0IsKill);
3034 }
3035 return 0;
3036}
3037
3038unsigned fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3039 switch (VT.SimpleTy) {
3040 case MVT::v8i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
3041 case MVT::v4i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
3042 case MVT::v2i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
3043 default: return 0;
3044 }
3045}
3046
3047// Top-level FastEmit function.
3048
3049unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill) override {
3050 switch (Opcode) {
3051 case ARMISD::CALL: return fastEmit_ARMISD_CALL_r(VT, RetVT, Op0, Op0IsKill);
3052 case ARMISD::CALL_NOLINK: return fastEmit_ARMISD_CALL_NOLINK_r(VT, RetVT, Op0, Op0IsKill);
3053 case ARMISD::CALL_PRED: return fastEmit_ARMISD_CALL_PRED_r(VT, RetVT, Op0, Op0IsKill);
3054 case ARMISD::CMPFPEw0: return fastEmit_ARMISD_CMPFPEw0_r(VT, RetVT, Op0, Op0IsKill);
3055 case ARMISD::CMPFPw0: return fastEmit_ARMISD_CMPFPw0_r(VT, RetVT, Op0, Op0IsKill);
3056 case ARMISD::RRX: return fastEmit_ARMISD_RRX_r(VT, RetVT, Op0, Op0IsKill);
3057 case ARMISD::SRA_FLAG: return fastEmit_ARMISD_SRA_FLAG_r(VT, RetVT, Op0, Op0IsKill);
3058 case ARMISD::SRL_FLAG: return fastEmit_ARMISD_SRL_FLAG_r(VT, RetVT, Op0, Op0IsKill);
3059 case ARMISD::TC_RETURN: return fastEmit_ARMISD_TC_RETURN_r(VT, RetVT, Op0, Op0IsKill);
3060 case ARMISD::VADDVs: return fastEmit_ARMISD_VADDVs_r(VT, RetVT, Op0, Op0IsKill);
3061 case ARMISD::VADDVu: return fastEmit_ARMISD_VADDVu_r(VT, RetVT, Op0, Op0IsKill);
3062 case ARMISD::VDUP: return fastEmit_ARMISD_VDUP_r(VT, RetVT, Op0, Op0IsKill);
3063 case ARMISD::VMOVSR: return fastEmit_ARMISD_VMOVSR_r(VT, RetVT, Op0, Op0IsKill);
3064 case ARMISD::VMOVhr: return fastEmit_ARMISD_VMOVhr_r(VT, RetVT, Op0, Op0IsKill);
3065 case ARMISD::VMOVrh: return fastEmit_ARMISD_VMOVrh_r(VT, RetVT, Op0, Op0IsKill);
3066 case ARMISD::VREV16: return fastEmit_ARMISD_VREV16_r(VT, RetVT, Op0, Op0IsKill);
3067 case ARMISD::VREV32: return fastEmit_ARMISD_VREV32_r(VT, RetVT, Op0, Op0IsKill);
3068 case ARMISD::VREV64: return fastEmit_ARMISD_VREV64_r(VT, RetVT, Op0, Op0IsKill);
3069 case ARMISD::WIN__DBZCHK: return fastEmit_ARMISD_WIN__DBZCHK_r(VT, RetVT, Op0, Op0IsKill);
3070 case ARMISD::tSECALL: return fastEmit_ARMISD_tSECALL_r(VT, RetVT, Op0, Op0IsKill);
3071 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0, Op0IsKill);
3072 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
3073 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0, Op0IsKill);
3074 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0, Op0IsKill);
3075 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0, Op0IsKill);
3076 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0, Op0IsKill);
3077 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0, Op0IsKill);
3078 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0, Op0IsKill);
3079 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0, Op0IsKill);
3080 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0, Op0IsKill);
3081 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0, Op0IsKill);
3082 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0, Op0IsKill);
3083 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0, Op0IsKill);
3084 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
3085 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0, Op0IsKill);
3086 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0, Op0IsKill);
3087 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0, Op0IsKill);
3088 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0, Op0IsKill);
3089 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0, Op0IsKill);
3090 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0, Op0IsKill);
3091 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0, Op0IsKill);
3092 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
3093 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
3094 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0, Op0IsKill);
3095 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
3096 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0, Op0IsKill);
3097 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
3098 default: return 0;
3099 }
3100}
3101
3102// FastEmit functions for ARMISD::CMP.
3103
3104unsigned fastEmit_ARMISD_CMP_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3105 if (RetVT.SimpleTy != MVT::isVoid)
3106 return 0;
3107 if ((Subtarget->isThumb2())) {
3108 return fastEmitInst_rr(ARM::t2CMPrr, &ARM::GPRnopcRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3109 }
3110 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
3111 return fastEmitInst_rr(ARM::tCMPr, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3112 }
3113 if ((!Subtarget->isThumb())) {
3114 return fastEmitInst_rr(ARM::CMPrr, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3115 }
3116 return 0;
3117}
3118
3119unsigned fastEmit_ARMISD_CMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3120 switch (VT.SimpleTy) {
3121 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3122 default: return 0;
3123 }
3124}
3125
3126// FastEmit functions for ARMISD::CMPFP.
3127
3128unsigned fastEmit_ARMISD_CMPFP_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3129 if (RetVT.SimpleTy != MVT::isVoid)
3130 return 0;
3131 if ((Subtarget->hasFullFP16())) {
3132 return fastEmitInst_rr(ARM::VCMPH, &ARM::HPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3133 }
3134 return 0;
3135}
3136
3137unsigned fastEmit_ARMISD_CMPFP_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3138 if (RetVT.SimpleTy != MVT::isVoid)
3139 return 0;
3140 if ((Subtarget->hasVFP2Base())) {
3141 return fastEmitInst_rr(ARM::VCMPS, &ARM::SPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3142 }
3143 return 0;
3144}
3145
3146unsigned fastEmit_ARMISD_CMPFP_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3147 if (RetVT.SimpleTy != MVT::isVoid)
3148 return 0;
3149 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3150 return fastEmitInst_rr(ARM::VCMPD, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3151 }
3152 return 0;
3153}
3154
3155unsigned fastEmit_ARMISD_CMPFP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3156 switch (VT.SimpleTy) {
3157 case MVT::f16: return fastEmit_ARMISD_CMPFP_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3158 case MVT::f32: return fastEmit_ARMISD_CMPFP_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3159 case MVT::f64: return fastEmit_ARMISD_CMPFP_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3160 default: return 0;
3161 }
3162}
3163
3164// FastEmit functions for ARMISD::CMPFPE.
3165
3166unsigned fastEmit_ARMISD_CMPFPE_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3167 if (RetVT.SimpleTy != MVT::isVoid)
3168 return 0;
3169 if ((Subtarget->hasFullFP16())) {
3170 return fastEmitInst_rr(ARM::VCMPEH, &ARM::HPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3171 }
3172 return 0;
3173}
3174
3175unsigned fastEmit_ARMISD_CMPFPE_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3176 if (RetVT.SimpleTy != MVT::isVoid)
3177 return 0;
3178 if ((Subtarget->hasVFP2Base())) {
3179 return fastEmitInst_rr(ARM::VCMPES, &ARM::SPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3180 }
3181 return 0;
3182}
3183
3184unsigned fastEmit_ARMISD_CMPFPE_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3185 if (RetVT.SimpleTy != MVT::isVoid)
3186 return 0;
3187 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
3188 return fastEmitInst_rr(ARM::VCMPED, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3189 }
3190 return 0;
3191}
3192
3193unsigned fastEmit_ARMISD_CMPFPE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3194 switch (VT.SimpleTy) {
3195 case MVT::f16: return fastEmit_ARMISD_CMPFPE_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3196 case MVT::f32: return fastEmit_ARMISD_CMPFPE_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3197 case MVT::f64: return fastEmit_ARMISD_CMPFPE_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3198 default: return 0;
3199 }
3200}
3201
3202// FastEmit functions for ARMISD::CMPZ.
3203
3204unsigned fastEmit_ARMISD_CMPZ_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3205 if (RetVT.SimpleTy != MVT::isVoid)
3206 return 0;
3207 if ((Subtarget->isThumb2())) {
3208 return fastEmitInst_rr(ARM::t2CMPrr, &ARM::GPRnopcRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3209 }
3210 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
3211 return fastEmitInst_rr(ARM::tCMPr, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3212 }
3213 if ((!Subtarget->isThumb())) {
3214 return fastEmitInst_rr(ARM::CMPrr, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3215 }
3216 return 0;
3217}
3218
3219unsigned fastEmit_ARMISD_CMPZ_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3220 switch (VT.SimpleTy) {
3221 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3222 default: return 0;
3223 }
3224}
3225
3226// FastEmit functions for ARMISD::EH_SJLJ_LONGJMP.
3227
3228unsigned fastEmit_ARMISD_EH_SJLJ_LONGJMP_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3229 if (RetVT.SimpleTy != MVT::isVoid)
3230 return 0;
3231 if ((Subtarget->isThumb()) && (Subtarget->isTargetWindows())) {
3232 return fastEmitInst_rr(ARM::tInt_WIN_eh_sjlj_longjmp, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3233 }
3234 if ((!Subtarget->isTargetWindows()) && (Subtarget->isThumb())) {
3235 return fastEmitInst_rr(ARM::tInt_eh_sjlj_longjmp, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3236 }
3237 if ((!Subtarget->isThumb())) {
3238 return fastEmitInst_rr(ARM::Int_eh_sjlj_longjmp, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3239 }
3240 return 0;
3241}
3242
3243unsigned fastEmit_ARMISD_EH_SJLJ_LONGJMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3244 switch (VT.SimpleTy) {
3245 case MVT::i32: return fastEmit_ARMISD_EH_SJLJ_LONGJMP_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3246 default: return 0;
3247 }
3248}
3249
3250// FastEmit functions for ARMISD::EH_SJLJ_SETJMP.
3251
3252unsigned fastEmit_ARMISD_EH_SJLJ_SETJMP_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3253 if (RetVT.SimpleTy != MVT::i32)
3254 return 0;
3255 if ((Subtarget->isThumb2()) && (!Subtarget->hasVFP2Base())) {
3256 return fastEmitInst_rr(ARM::t2Int_eh_sjlj_setjmp_nofp, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3257 }
3258 if ((Subtarget->hasVFP2Base()) && (Subtarget->isThumb2())) {
3259 return fastEmitInst_rr(ARM::t2Int_eh_sjlj_setjmp, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3260 }
3261 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
3262 return fastEmitInst_rr(ARM::tInt_eh_sjlj_setjmp, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3263 }
3264 if ((!Subtarget->isThumb()) && (!Subtarget->hasVFP2Base())) {
3265 return fastEmitInst_rr(ARM::Int_eh_sjlj_setjmp_nofp, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3266 }
3267 if ((Subtarget->hasVFP2Base()) && (!Subtarget->isThumb())) {
3268 return fastEmitInst_rr(ARM::Int_eh_sjlj_setjmp, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3269 }
3270 return 0;
3271}
3272
3273unsigned fastEmit_ARMISD_EH_SJLJ_SETJMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3274 switch (VT.SimpleTy) {
3275 case MVT::i32: return fastEmit_ARMISD_EH_SJLJ_SETJMP_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3276 default: return 0;
3277 }
3278}
3279
3280// FastEmit functions for ARMISD::QADD16b.
3281
3282unsigned fastEmit_ARMISD_QADD16b_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3283 if (RetVT.SimpleTy != MVT::i32)
3284 return 0;
3285 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3286 return fastEmitInst_rr(ARM::t2QADD16, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3287 }
3288 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3289 return fastEmitInst_rr(ARM::QADD16, &ARM::GPRnopcRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3290 }
3291 return 0;
3292}
3293
3294unsigned fastEmit_ARMISD_QADD16b_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3295 switch (VT.SimpleTy) {
3296 case MVT::i32: return fastEmit_ARMISD_QADD16b_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3297 default: return 0;
3298 }
3299}
3300
3301// FastEmit functions for ARMISD::QADD8b.
3302
3303unsigned fastEmit_ARMISD_QADD8b_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3304 if (RetVT.SimpleTy != MVT::i32)
3305 return 0;
3306 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3307 return fastEmitInst_rr(ARM::t2QADD8, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3308 }
3309 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3310 return fastEmitInst_rr(ARM::QADD8, &ARM::GPRnopcRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3311 }
3312 return 0;
3313}
3314
3315unsigned fastEmit_ARMISD_QADD8b_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3316 switch (VT.SimpleTy) {
3317 case MVT::i32: return fastEmit_ARMISD_QADD8b_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3318 default: return 0;
3319 }
3320}
3321
3322// FastEmit functions for ARMISD::QSUB16b.
3323
3324unsigned fastEmit_ARMISD_QSUB16b_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3325 if (RetVT.SimpleTy != MVT::i32)
3326 return 0;
3327 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3328 return fastEmitInst_rr(ARM::t2QSUB16, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3329 }
3330 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3331 return fastEmitInst_rr(ARM::QSUB16, &ARM::GPRnopcRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3332 }
3333 return 0;
3334}
3335
3336unsigned fastEmit_ARMISD_QSUB16b_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3337 switch (VT.SimpleTy) {
3338 case MVT::i32: return fastEmit_ARMISD_QSUB16b_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3339 default: return 0;
3340 }
3341}
3342
3343// FastEmit functions for ARMISD::QSUB8b.
3344
3345unsigned fastEmit_ARMISD_QSUB8b_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3346 if (RetVT.SimpleTy != MVT::i32)
3347 return 0;
3348 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3349 return fastEmitInst_rr(ARM::t2QSUB8, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3350 }
3351 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
3352 return fastEmitInst_rr(ARM::QSUB8, &ARM::GPRnopcRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3353 }
3354 return 0;
3355}
3356
3357unsigned fastEmit_ARMISD_QSUB8b_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3358 switch (VT.SimpleTy) {
3359 case MVT::i32: return fastEmit_ARMISD_QSUB8b_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3360 default: return 0;
3361 }
3362}
3363
3364// FastEmit functions for ARMISD::SMULWB.
3365
3366unsigned fastEmit_ARMISD_SMULWB_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3367 if (RetVT.SimpleTy != MVT::i32)
3368 return 0;
3369 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3370 return fastEmitInst_rr(ARM::t2SMULWB, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3371 }
3372 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
3373 return fastEmitInst_rr(ARM::SMULWB, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3374 }
3375 return 0;
3376}
3377
3378unsigned fastEmit_ARMISD_SMULWB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3379 switch (VT.SimpleTy) {
3380 case MVT::i32: return fastEmit_ARMISD_SMULWB_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3381 default: return 0;
3382 }
3383}
3384
3385// FastEmit functions for ARMISD::SMULWT.
3386
3387unsigned fastEmit_ARMISD_SMULWT_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3388 if (RetVT.SimpleTy != MVT::i32)
3389 return 0;
3390 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
3391 return fastEmitInst_rr(ARM::t2SMULWT, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3392 }
3393 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
3394 return fastEmitInst_rr(ARM::SMULWT, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3395 }
3396 return 0;
3397}
3398
3399unsigned fastEmit_ARMISD_SMULWT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3400 switch (VT.SimpleTy) {
3401 case MVT::i32: return fastEmit_ARMISD_SMULWT_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3402 default: return 0;
3403 }
3404}
3405
3406// FastEmit functions for ARMISD::SUBS.
3407
3408unsigned fastEmit_ARMISD_SUBS_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3409 if (RetVT.SimpleTy != MVT::i32)
3410 return 0;
3411 if ((Subtarget->isThumb2())) {
3412 return fastEmitInst_rr(ARM::t2SUBSrr, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3413 }
3414 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
3415 return fastEmitInst_rr(ARM::tSUBSrr, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3416 }
3417 if ((!Subtarget->isThumb())) {
3418 return fastEmitInst_rr(ARM::SUBSrr, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3419 }
3420 return 0;
3421}
3422
3423unsigned fastEmit_ARMISD_SUBS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3424 switch (VT.SimpleTy) {
3425 case MVT::i32: return fastEmit_ARMISD_SUBS_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3426 default: return 0;
3427 }
3428}
3429
3430// FastEmit functions for ARMISD::VMLAVs.
3431
3432unsigned fastEmit_ARMISD_VMLAVs_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3433 if (RetVT.SimpleTy != MVT::i32)
3434 return 0;
3435 if ((Subtarget->hasMVEIntegerOps())) {
3436 return fastEmitInst_rr(ARM::MVE_VMLADAVs8, &ARM::tGPREvenRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3437 }
3438 return 0;
3439}
3440
3441unsigned fastEmit_ARMISD_VMLAVs_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3442 if (RetVT.SimpleTy != MVT::i32)
3443 return 0;
3444 if ((Subtarget->hasMVEIntegerOps())) {
3445 return fastEmitInst_rr(ARM::MVE_VMLADAVs16, &ARM::tGPREvenRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3446 }
3447 return 0;
3448}
3449
3450unsigned fastEmit_ARMISD_VMLAVs_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3451 switch (VT.SimpleTy) {
3452 case MVT::v16i8: return fastEmit_ARMISD_VMLAVs_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3453 case MVT::v8i16: return fastEmit_ARMISD_VMLAVs_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3454 default: return 0;
3455 }
3456}
3457
3458// FastEmit functions for ARMISD::VMLAVu.
3459
3460unsigned fastEmit_ARMISD_VMLAVu_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3461 if (RetVT.SimpleTy != MVT::i32)
3462 return 0;
3463 if ((Subtarget->hasMVEIntegerOps())) {
3464 return fastEmitInst_rr(ARM::MVE_VMLADAVu8, &ARM::tGPREvenRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3465 }
3466 return 0;
3467}
3468
3469unsigned fastEmit_ARMISD_VMLAVu_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3470 if (RetVT.SimpleTy != MVT::i32)
3471 return 0;
3472 if ((Subtarget->hasMVEIntegerOps())) {
3473 return fastEmitInst_rr(ARM::MVE_VMLADAVu16, &ARM::tGPREvenRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3474 }
3475 return 0;
3476}
3477
3478unsigned fastEmit_ARMISD_VMLAVu_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3479 switch (VT.SimpleTy) {
3480 case MVT::v16i8: return fastEmit_ARMISD_VMLAVu_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3481 case MVT::v8i16: return fastEmit_ARMISD_VMLAVu_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3482 default: return 0;
3483 }
3484}
3485
3486// FastEmit functions for ARMISD::VMOVDRR.
3487
3488unsigned fastEmit_ARMISD_VMOVDRR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3489 if (RetVT.SimpleTy != MVT::f64)
3490 return 0;
3491 if ((Subtarget->hasFPRegs())) {
3492 return fastEmitInst_rr(ARM::VMOVDRR, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3493 }
3494 return 0;
3495}
3496
3497unsigned fastEmit_ARMISD_VMOVDRR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3498 switch (VT.SimpleTy) {
3499 case MVT::i32: return fastEmit_ARMISD_VMOVDRR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3500 default: return 0;
3501 }
3502}
3503
3504// FastEmit functions for ARMISD::VMULLs.
3505
3506unsigned fastEmit_ARMISD_VMULLs_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3507 if (RetVT.SimpleTy != MVT::v8i16)
3508 return 0;
3509 if ((Subtarget->hasNEON())) {
3510 return fastEmitInst_rr(ARM::VMULLsv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3511 }
3512 return 0;
3513}
3514
3515unsigned fastEmit_ARMISD_VMULLs_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3516 if (RetVT.SimpleTy != MVT::v4i32)
3517 return 0;
3518 if ((Subtarget->hasNEON())) {
3519 return fastEmitInst_rr(ARM::VMULLsv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3520 }
3521 return 0;
3522}
3523
3524unsigned fastEmit_ARMISD_VMULLs_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3525 if (RetVT.SimpleTy != MVT::v2i64)
3526 return 0;
3527 if ((Subtarget->hasNEON())) {
3528 return fastEmitInst_rr(ARM::VMULLsv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3529 }
3530 return 0;
3531}
3532
3533unsigned fastEmit_ARMISD_VMULLs_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3534 if (RetVT.SimpleTy != MVT::v2i64)
3535 return 0;
3536 if ((Subtarget->hasMVEIntegerOps())) {
3537 return fastEmitInst_rr(ARM::MVE_VMULLBs32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3538 }
3539 return 0;
3540}
3541
3542unsigned fastEmit_ARMISD_VMULLs_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3543 switch (VT.SimpleTy) {
3544 case MVT::v8i8: return fastEmit_ARMISD_VMULLs_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3545 case MVT::v4i16: return fastEmit_ARMISD_VMULLs_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3546 case MVT::v2i32: return fastEmit_ARMISD_VMULLs_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3547 case MVT::v4i32: return fastEmit_ARMISD_VMULLs_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3548 default: return 0;
3549 }
3550}
3551
3552// FastEmit functions for ARMISD::VMULLu.
3553
3554unsigned fastEmit_ARMISD_VMULLu_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3555 if (RetVT.SimpleTy != MVT::v8i16)
3556 return 0;
3557 if ((Subtarget->hasNEON())) {
3558 return fastEmitInst_rr(ARM::VMULLuv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3559 }
3560 return 0;
3561}
3562
3563unsigned fastEmit_ARMISD_VMULLu_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3564 if (RetVT.SimpleTy != MVT::v4i32)
3565 return 0;
3566 if ((Subtarget->hasNEON())) {
3567 return fastEmitInst_rr(ARM::VMULLuv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3568 }
3569 return 0;
3570}
3571
3572unsigned fastEmit_ARMISD_VMULLu_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3573 if (RetVT.SimpleTy != MVT::v2i64)
3574 return 0;
3575 if ((Subtarget->hasNEON())) {
3576 return fastEmitInst_rr(ARM::VMULLuv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3577 }
3578 return 0;
3579}
3580
3581unsigned fastEmit_ARMISD_VMULLu_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3582 if (RetVT.SimpleTy != MVT::v2i64)
3583 return 0;
3584 if ((Subtarget->hasMVEIntegerOps())) {
3585 return fastEmitInst_rr(ARM::MVE_VMULLBu32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3586 }
3587 return 0;
3588}
3589
3590unsigned fastEmit_ARMISD_VMULLu_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3591 switch (VT.SimpleTy) {
3592 case MVT::v8i8: return fastEmit_ARMISD_VMULLu_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3593 case MVT::v4i16: return fastEmit_ARMISD_VMULLu_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3594 case MVT::v2i32: return fastEmit_ARMISD_VMULLu_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3595 case MVT::v4i32: return fastEmit_ARMISD_VMULLu_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3596 default: return 0;
3597 }
3598}
3599
3600// FastEmit functions for ARMISD::VQDMULH.
3601
3602unsigned fastEmit_ARMISD_VQDMULH_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3603 if (RetVT.SimpleTy != MVT::v16i8)
3604 return 0;
3605 if ((Subtarget->hasMVEIntegerOps())) {
3606 return fastEmitInst_rr(ARM::MVE_VQDMULHi8, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3607 }
3608 return 0;
3609}
3610
3611unsigned fastEmit_ARMISD_VQDMULH_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3612 if (RetVT.SimpleTy != MVT::v8i16)
3613 return 0;
3614 if ((Subtarget->hasMVEIntegerOps())) {
3615 return fastEmitInst_rr(ARM::MVE_VQDMULHi16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3616 }
3617 return 0;
3618}
3619
3620unsigned fastEmit_ARMISD_VQDMULH_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3621 if (RetVT.SimpleTy != MVT::v4i32)
3622 return 0;
3623 if ((Subtarget->hasMVEIntegerOps())) {
3624 return fastEmitInst_rr(ARM::MVE_VQDMULHi32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3625 }
3626 return 0;
3627}
3628
3629unsigned fastEmit_ARMISD_VQDMULH_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3630 switch (VT.SimpleTy) {
3631 case MVT::v16i8: return fastEmit_ARMISD_VQDMULH_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3632 case MVT::v8i16: return fastEmit_ARMISD_VQDMULH_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3633 case MVT::v4i32: return fastEmit_ARMISD_VQDMULH_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3634 default: return 0;
3635 }
3636}
3637
3638// FastEmit functions for ARMISD::VSHLs.
3639
3640unsigned fastEmit_ARMISD_VSHLs_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3641 if (RetVT.SimpleTy != MVT::v8i8)
3642 return 0;
3643 if ((Subtarget->hasNEON())) {
3644 return fastEmitInst_rr(ARM::VSHLsv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3645 }
3646 return 0;
3647}
3648
3649unsigned fastEmit_ARMISD_VSHLs_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3650 if (RetVT.SimpleTy != MVT::v16i8)
3651 return 0;
3652 if ((Subtarget->hasMVEIntegerOps())) {
3653 return fastEmitInst_rr(ARM::MVE_VSHL_by_vecs8, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3654 }
3655 if ((Subtarget->hasNEON())) {
3656 return fastEmitInst_rr(ARM::VSHLsv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3657 }
3658 return 0;
3659}
3660
3661unsigned fastEmit_ARMISD_VSHLs_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3662 if (RetVT.SimpleTy != MVT::v4i16)
3663 return 0;
3664 if ((Subtarget->hasNEON())) {
3665 return fastEmitInst_rr(ARM::VSHLsv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3666 }
3667 return 0;
3668}
3669
3670unsigned fastEmit_ARMISD_VSHLs_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3671 if (RetVT.SimpleTy != MVT::v8i16)
3672 return 0;
3673 if ((Subtarget->hasMVEIntegerOps())) {
3674 return fastEmitInst_rr(ARM::MVE_VSHL_by_vecs16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3675 }
3676 if ((Subtarget->hasNEON())) {
3677 return fastEmitInst_rr(ARM::VSHLsv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3678 }
3679 return 0;
3680}
3681
3682unsigned fastEmit_ARMISD_VSHLs_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3683 if (RetVT.SimpleTy != MVT::v2i32)
3684 return 0;
3685 if ((Subtarget->hasNEON())) {
3686 return fastEmitInst_rr(ARM::VSHLsv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3687 }
3688 return 0;
3689}
3690
3691unsigned fastEmit_ARMISD_VSHLs_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3692 if (RetVT.SimpleTy != MVT::v4i32)
3693 return 0;
3694 if ((Subtarget->hasMVEIntegerOps())) {
3695 return fastEmitInst_rr(ARM::MVE_VSHL_by_vecs32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3696 }
3697 if ((Subtarget->hasNEON())) {
3698 return fastEmitInst_rr(ARM::VSHLsv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3699 }
3700 return 0;
3701}
3702
3703unsigned fastEmit_ARMISD_VSHLs_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3704 if (RetVT.SimpleTy != MVT::v1i64)
3705 return 0;
3706 if ((Subtarget->hasNEON())) {
3707 return fastEmitInst_rr(ARM::VSHLsv1i64, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3708 }
3709 return 0;
3710}
3711
3712unsigned fastEmit_ARMISD_VSHLs_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3713 if (RetVT.SimpleTy != MVT::v2i64)
3714 return 0;
3715 if ((Subtarget->hasNEON())) {
3716 return fastEmitInst_rr(ARM::VSHLsv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3717 }
3718 return 0;
3719}
3720
3721unsigned fastEmit_ARMISD_VSHLs_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3722 switch (VT.SimpleTy) {
3723 case MVT::v8i8: return fastEmit_ARMISD_VSHLs_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3724 case MVT::v16i8: return fastEmit_ARMISD_VSHLs_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3725 case MVT::v4i16: return fastEmit_ARMISD_VSHLs_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3726 case MVT::v8i16: return fastEmit_ARMISD_VSHLs_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3727 case MVT::v2i32: return fastEmit_ARMISD_VSHLs_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3728 case MVT::v4i32: return fastEmit_ARMISD_VSHLs_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3729 case MVT::v1i64: return fastEmit_ARMISD_VSHLs_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3730 case MVT::v2i64: return fastEmit_ARMISD_VSHLs_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3731 default: return 0;
3732 }
3733}
3734
3735// FastEmit functions for ARMISD::VSHLu.
3736
3737unsigned fastEmit_ARMISD_VSHLu_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3738 if (RetVT.SimpleTy != MVT::v8i8)
3739 return 0;
3740 if ((Subtarget->hasNEON())) {
3741 return fastEmitInst_rr(ARM::VSHLuv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3742 }
3743 return 0;
3744}
3745
3746unsigned fastEmit_ARMISD_VSHLu_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3747 if (RetVT.SimpleTy != MVT::v16i8)
3748 return 0;
3749 if ((Subtarget->hasMVEIntegerOps())) {
3750 return fastEmitInst_rr(ARM::MVE_VSHL_by_vecu8, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3751 }
3752 if ((Subtarget->hasNEON())) {
3753 return fastEmitInst_rr(ARM::VSHLuv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3754 }
3755 return 0;
3756}
3757
3758unsigned fastEmit_ARMISD_VSHLu_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3759 if (RetVT.SimpleTy != MVT::v4i16)
3760 return 0;
3761 if ((Subtarget->hasNEON())) {
3762 return fastEmitInst_rr(ARM::VSHLuv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3763 }
3764 return 0;
3765}
3766
3767unsigned fastEmit_ARMISD_VSHLu_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3768 if (RetVT.SimpleTy != MVT::v8i16)
3769 return 0;
3770 if ((Subtarget->hasMVEIntegerOps())) {
3771 return fastEmitInst_rr(ARM::MVE_VSHL_by_vecu16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3772 }
3773 if ((Subtarget->hasNEON())) {
3774 return fastEmitInst_rr(ARM::VSHLuv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3775 }
3776 return 0;
3777}
3778
3779unsigned fastEmit_ARMISD_VSHLu_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3780 if (RetVT.SimpleTy != MVT::v2i32)
3781 return 0;
3782 if ((Subtarget->hasNEON())) {
3783 return fastEmitInst_rr(ARM::VSHLuv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3784 }
3785 return 0;
3786}
3787
3788unsigned fastEmit_ARMISD_VSHLu_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3789 if (RetVT.SimpleTy != MVT::v4i32)
3790 return 0;
3791 if ((Subtarget->hasMVEIntegerOps())) {
3792 return fastEmitInst_rr(ARM::MVE_VSHL_by_vecu32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3793 }
3794 if ((Subtarget->hasNEON())) {
3795 return fastEmitInst_rr(ARM::VSHLuv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3796 }
3797 return 0;
3798}
3799
3800unsigned fastEmit_ARMISD_VSHLu_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3801 if (RetVT.SimpleTy != MVT::v1i64)
3802 return 0;
3803 if ((Subtarget->hasNEON())) {
3804 return fastEmitInst_rr(ARM::VSHLuv1i64, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3805 }
3806 return 0;
3807}
3808
3809unsigned fastEmit_ARMISD_VSHLu_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3810 if (RetVT.SimpleTy != MVT::v2i64)
3811 return 0;
3812 if ((Subtarget->hasNEON())) {
3813 return fastEmitInst_rr(ARM::VSHLuv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3814 }
3815 return 0;
3816}
3817
3818unsigned fastEmit_ARMISD_VSHLu_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3819 switch (VT.SimpleTy) {
3820 case MVT::v8i8: return fastEmit_ARMISD_VSHLu_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3821 case MVT::v16i8: return fastEmit_ARMISD_VSHLu_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3822 case MVT::v4i16: return fastEmit_ARMISD_VSHLu_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3823 case MVT::v8i16: return fastEmit_ARMISD_VSHLu_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3824 case MVT::v2i32: return fastEmit_ARMISD_VSHLu_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3825 case MVT::v4i32: return fastEmit_ARMISD_VSHLu_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3826 case MVT::v1i64: return fastEmit_ARMISD_VSHLu_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3827 case MVT::v2i64: return fastEmit_ARMISD_VSHLu_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3828 default: return 0;
3829 }
3830}
3831
3832// FastEmit functions for ARMISD::VTBL1.
3833
3834unsigned fastEmit_ARMISD_VTBL1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3835 if (RetVT.SimpleTy != MVT::v8i8)
3836 return 0;
3837 if ((Subtarget->hasNEON())) {
3838 return fastEmitInst_rr(ARM::VTBL1, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3839 }
3840 return 0;
3841}
3842
3843unsigned fastEmit_ARMISD_VTBL1_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3844 switch (VT.SimpleTy) {
3845 case MVT::v8i8: return fastEmit_ARMISD_VTBL1_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3846 default: return 0;
3847 }
3848}
3849
3850// FastEmit functions for ARMISD::VTST.
3851
3852unsigned fastEmit_ARMISD_VTST_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3853 if (RetVT.SimpleTy != MVT::v8i8)
3854 return 0;
3855 if ((Subtarget->hasNEON())) {
3856 return fastEmitInst_rr(ARM::VTSTv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3857 }
3858 return 0;
3859}
3860
3861unsigned fastEmit_ARMISD_VTST_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3862 if (RetVT.SimpleTy != MVT::v16i8)
3863 return 0;
3864 if ((Subtarget->hasNEON())) {
3865 return fastEmitInst_rr(ARM::VTSTv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3866 }
3867 return 0;
3868}
3869
3870unsigned fastEmit_ARMISD_VTST_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3871 if (RetVT.SimpleTy != MVT::v4i16)
3872 return 0;
3873 if ((Subtarget->hasNEON())) {
3874 return fastEmitInst_rr(ARM::VTSTv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3875 }
3876 return 0;
3877}
3878
3879unsigned fastEmit_ARMISD_VTST_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3880 if (RetVT.SimpleTy != MVT::v8i16)
3881 return 0;
3882 if ((Subtarget->hasNEON())) {
3883 return fastEmitInst_rr(ARM::VTSTv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3884 }
3885 return 0;
3886}
3887
3888unsigned fastEmit_ARMISD_VTST_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3889 if (RetVT.SimpleTy != MVT::v2i32)
3890 return 0;
3891 if ((Subtarget->hasNEON())) {
3892 return fastEmitInst_rr(ARM::VTSTv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3893 }
3894 return 0;
3895}
3896
3897unsigned fastEmit_ARMISD_VTST_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3898 if (RetVT.SimpleTy != MVT::v4i32)
3899 return 0;
3900 if ((Subtarget->hasNEON())) {
3901 return fastEmitInst_rr(ARM::VTSTv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3902 }
3903 return 0;
3904}
3905
3906unsigned fastEmit_ARMISD_VTST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3907 switch (VT.SimpleTy) {
3908 case MVT::v8i8: return fastEmit_ARMISD_VTST_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3909 case MVT::v16i8: return fastEmit_ARMISD_VTST_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3910 case MVT::v4i16: return fastEmit_ARMISD_VTST_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3911 case MVT::v8i16: return fastEmit_ARMISD_VTST_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3912 case MVT::v2i32: return fastEmit_ARMISD_VTST_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3913 case MVT::v4i32: return fastEmit_ARMISD_VTST_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
3914 default: return 0;
3915 }
3916}
3917
3918// FastEmit functions for ISD::ADD.
3919
3920unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3921 if (RetVT.SimpleTy != MVT::i32)
3922 return 0;
3923 if ((Subtarget->isThumb2())) {
3924 return fastEmitInst_rr(ARM::t2ADDrr, &ARM::GPRnopcRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3925 }
3926 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
3927 return fastEmitInst_rr(ARM::tADDrr, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3928 }
3929 if ((!Subtarget->isThumb())) {
3930 return fastEmitInst_rr(ARM::ADDrr, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3931 }
3932 return 0;
3933}
3934
3935unsigned fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3936 if (RetVT.SimpleTy != MVT::v8i8)
3937 return 0;
3938 if ((Subtarget->hasNEON())) {
3939 return fastEmitInst_rr(ARM::VADDv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3940 }
3941 return 0;
3942}
3943
3944unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3945 if (RetVT.SimpleTy != MVT::v16i8)
3946 return 0;
3947 if ((Subtarget->hasMVEIntegerOps())) {
3948 return fastEmitInst_rr(ARM::MVE_VADDi8, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3949 }
3950 if ((Subtarget->hasNEON())) {
3951 return fastEmitInst_rr(ARM::VADDv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3952 }
3953 return 0;
3954}
3955
3956unsigned fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3957 if (RetVT.SimpleTy != MVT::v4i16)
3958 return 0;
3959 if ((Subtarget->hasNEON())) {
3960 return fastEmitInst_rr(ARM::VADDv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3961 }
3962 return 0;
3963}
3964
3965unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3966 if (RetVT.SimpleTy != MVT::v8i16)
3967 return 0;
3968 if ((Subtarget->hasMVEIntegerOps())) {
3969 return fastEmitInst_rr(ARM::MVE_VADDi16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3970 }
3971 if ((Subtarget->hasNEON())) {
3972 return fastEmitInst_rr(ARM::VADDv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3973 }
3974 return 0;
3975}
3976
3977unsigned fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3978 if (RetVT.SimpleTy != MVT::v2i32)
3979 return 0;
3980 if ((Subtarget->hasNEON())) {
3981 return fastEmitInst_rr(ARM::VADDv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3982 }
3983 return 0;
3984}
3985
3986unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3987 if (RetVT.SimpleTy != MVT::v4i32)
3988 return 0;
3989 if ((Subtarget->hasMVEIntegerOps())) {
3990 return fastEmitInst_rr(ARM::MVE_VADDi32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3991 }
3992 if ((Subtarget->hasNEON())) {
3993 return fastEmitInst_rr(ARM::VADDv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
3994 }
3995 return 0;
3996}
3997
3998unsigned fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
3999 if (RetVT.SimpleTy != MVT::v1i64)
4000 return 0;
4001 if ((Subtarget->hasNEON())) {
4002 return fastEmitInst_rr(ARM::VADDv1i64, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4003 }
4004 return 0;
4005}
4006
4007unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4008 if (RetVT.SimpleTy != MVT::v2i64)
4009 return 0;
4010 if ((Subtarget->hasNEON())) {
4011 return fastEmitInst_rr(ARM::VADDv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4012 }
4013 return 0;
4014}
4015
4016unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4017 switch (VT.SimpleTy) {
4018 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4019 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4020 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4021 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4022 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4023 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4024 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4025 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4026 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4027 default: return 0;
4028 }
4029}
4030
4031// FastEmit functions for ISD::AND.
4032
4033unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4034 if (RetVT.SimpleTy != MVT::i32)
4035 return 0;
4036 if ((Subtarget->isThumb2())) {
4037 return fastEmitInst_rr(ARM::t2ANDrr, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4038 }
4039 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4040 return fastEmitInst_rr(ARM::tAND, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4041 }
4042 if ((!Subtarget->isThumb())) {
4043 return fastEmitInst_rr(ARM::ANDrr, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4044 }
4045 return 0;
4046}
4047
4048unsigned fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4049 if (RetVT.SimpleTy != MVT::v16i8)
4050 return 0;
4051 if ((Subtarget->hasMVEIntegerOps())) {
4052 return fastEmitInst_rr(ARM::MVE_VAND, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4053 }
4054 return 0;
4055}
4056
4057unsigned fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4058 if (RetVT.SimpleTy != MVT::v8i16)
4059 return 0;
4060 if ((Subtarget->hasMVEIntegerOps())) {
4061 return fastEmitInst_rr(ARM::MVE_VAND, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4062 }
4063 return 0;
4064}
4065
4066unsigned fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4067 if (RetVT.SimpleTy != MVT::v2i32)
4068 return 0;
4069 if ((Subtarget->hasNEON())) {
4070 return fastEmitInst_rr(ARM::VANDd, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4071 }
4072 return 0;
4073}
4074
4075unsigned fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4076 if (RetVT.SimpleTy != MVT::v4i32)
4077 return 0;
4078 if ((Subtarget->hasMVEIntegerOps())) {
4079 return fastEmitInst_rr(ARM::MVE_VAND, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4080 }
4081 if ((Subtarget->hasNEON())) {
4082 return fastEmitInst_rr(ARM::VANDq, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4083 }
4084 return 0;
4085}
4086
4087unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4088 if (RetVT.SimpleTy != MVT::v2i64)
4089 return 0;
4090 if ((Subtarget->hasMVEIntegerOps())) {
4091 return fastEmitInst_rr(ARM::MVE_VAND, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4092 }
4093 return 0;
4094}
4095
4096unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4097 switch (VT.SimpleTy) {
4098 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4099 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4100 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4101 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4102 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4103 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4104 default: return 0;
4105 }
4106}
4107
4108// FastEmit functions for ISD::FADD.
4109
4110unsigned fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4111 if (RetVT.SimpleTy != MVT::f16)
4112 return 0;
4113 if ((Subtarget->hasFullFP16())) {
4114 return fastEmitInst_rr(ARM::VADDH, &ARM::HPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4115 }
4116 return 0;
4117}
4118
4119unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4120 if (RetVT.SimpleTy != MVT::f32)
4121 return 0;
4122 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
4123 return fastEmitInst_rr(ARM::VADDS, &ARM::SPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4124 }
4125 return 0;
4126}
4127
4128unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4129 if (RetVT.SimpleTy != MVT::f64)
4130 return 0;
4131 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
4132 return fastEmitInst_rr(ARM::VADDD, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4133 }
4134 return 0;
4135}
4136
4137unsigned fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4138 if (RetVT.SimpleTy != MVT::v4f16)
4139 return 0;
4140 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4141 return fastEmitInst_rr(ARM::VADDhd, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4142 }
4143 return 0;
4144}
4145
4146unsigned fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4147 if (RetVT.SimpleTy != MVT::v8f16)
4148 return 0;
4149 if ((Subtarget->hasMVEFloatOps())) {
4150 return fastEmitInst_rr(ARM::MVE_VADDf16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4151 }
4152 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4153 return fastEmitInst_rr(ARM::VADDhq, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4154 }
4155 return 0;
4156}
4157
4158unsigned fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4159 if (RetVT.SimpleTy != MVT::v2f32)
4160 return 0;
4161 if ((Subtarget->hasNEON())) {
4162 return fastEmitInst_rr(ARM::VADDfd, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4163 }
4164 return 0;
4165}
4166
4167unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4168 if (RetVT.SimpleTy != MVT::v4f32)
4169 return 0;
4170 if ((Subtarget->hasMVEFloatOps())) {
4171 return fastEmitInst_rr(ARM::MVE_VADDf32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4172 }
4173 if ((Subtarget->hasNEON())) {
4174 return fastEmitInst_rr(ARM::VADDfq, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4175 }
4176 return 0;
4177}
4178
4179unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4180 switch (VT.SimpleTy) {
4181 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4182 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4183 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4184 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4185 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4186 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4187 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4188 default: return 0;
4189 }
4190}
4191
4192// FastEmit functions for ISD::FDIV.
4193
4194unsigned fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4195 if (RetVT.SimpleTy != MVT::f16)
4196 return 0;
4197 if ((Subtarget->hasFullFP16())) {
4198 return fastEmitInst_rr(ARM::VDIVH, &ARM::HPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4199 }
4200 return 0;
4201}
4202
4203unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4204 if (RetVT.SimpleTy != MVT::f32)
4205 return 0;
4206 if ((Subtarget->hasVFP2Base())) {
4207 return fastEmitInst_rr(ARM::VDIVS, &ARM::SPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4208 }
4209 return 0;
4210}
4211
4212unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4213 if (RetVT.SimpleTy != MVT::f64)
4214 return 0;
4215 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
4216 return fastEmitInst_rr(ARM::VDIVD, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4217 }
4218 return 0;
4219}
4220
4221unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4222 switch (VT.SimpleTy) {
4223 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4224 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4225 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4226 default: return 0;
4227 }
4228}
4229
4230// FastEmit functions for ISD::FMAXIMUM.
4231
4232unsigned fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4233 if (RetVT.SimpleTy != MVT::v4f16)
4234 return 0;
4235 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4236 return fastEmitInst_rr(ARM::VMAXhd, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4237 }
4238 return 0;
4239}
4240
4241unsigned fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4242 if (RetVT.SimpleTy != MVT::v8f16)
4243 return 0;
4244 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4245 return fastEmitInst_rr(ARM::VMAXhq, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4246 }
4247 return 0;
4248}
4249
4250unsigned fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4251 if (RetVT.SimpleTy != MVT::v2f32)
4252 return 0;
4253 if ((Subtarget->hasNEON())) {
4254 return fastEmitInst_rr(ARM::VMAXfd, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4255 }
4256 return 0;
4257}
4258
4259unsigned fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4260 if (RetVT.SimpleTy != MVT::v4f32)
4261 return 0;
4262 if ((Subtarget->hasNEON())) {
4263 return fastEmitInst_rr(ARM::VMAXfq, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4264 }
4265 return 0;
4266}
4267
4268unsigned fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4269 switch (VT.SimpleTy) {
4270 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4271 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4272 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4273 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4274 default: return 0;
4275 }
4276}
4277
4278// FastEmit functions for ISD::FMAXNUM.
4279
4280unsigned fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4281 if (RetVT.SimpleTy != MVT::f16)
4282 return 0;
4283 if ((Subtarget->hasFullFP16())) {
4284 return fastEmitInst_rr(ARM::VFP_VMAXNMH, &ARM::HPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4285 }
4286 return 0;
4287}
4288
4289unsigned fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4290 if (RetVT.SimpleTy != MVT::f32)
4291 return 0;
4292 if ((Subtarget->hasFPARMv8Base())) {
4293 return fastEmitInst_rr(ARM::VFP_VMAXNMS, &ARM::SPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4294 }
4295 return 0;
4296}
4297
4298unsigned fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4299 if (RetVT.SimpleTy != MVT::f64)
4300 return 0;
4301 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
4302 return fastEmitInst_rr(ARM::VFP_VMAXNMD, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4303 }
4304 return 0;
4305}
4306
4307unsigned fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4308 if (RetVT.SimpleTy != MVT::v4f16)
4309 return 0;
4310 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
4311 return fastEmitInst_rr(ARM::NEON_VMAXNMNDh, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4312 }
4313 return 0;
4314}
4315
4316unsigned fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4317 if (RetVT.SimpleTy != MVT::v8f16)
4318 return 0;
4319 if ((Subtarget->hasMVEFloatOps())) {
4320 return fastEmitInst_rr(ARM::MVE_VMAXNMf16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4321 }
4322 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
4323 return fastEmitInst_rr(ARM::NEON_VMAXNMNQh, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4324 }
4325 return 0;
4326}
4327
4328unsigned fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4329 if (RetVT.SimpleTy != MVT::v2f32)
4330 return 0;
4331 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
4332 return fastEmitInst_rr(ARM::NEON_VMAXNMNDf, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4333 }
4334 return 0;
4335}
4336
4337unsigned fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4338 if (RetVT.SimpleTy != MVT::v4f32)
4339 return 0;
4340 if ((Subtarget->hasMVEFloatOps())) {
4341 return fastEmitInst_rr(ARM::MVE_VMAXNMf32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4342 }
4343 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
4344 return fastEmitInst_rr(ARM::NEON_VMAXNMNQf, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4345 }
4346 return 0;
4347}
4348
4349unsigned fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4350 switch (VT.SimpleTy) {
4351 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4352 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4353 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4354 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4355 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4356 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4357 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4358 default: return 0;
4359 }
4360}
4361
4362// FastEmit functions for ISD::FMINIMUM.
4363
4364unsigned fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4365 if (RetVT.SimpleTy != MVT::v4f16)
4366 return 0;
4367 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4368 return fastEmitInst_rr(ARM::VMINhd, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4369 }
4370 return 0;
4371}
4372
4373unsigned fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4374 if (RetVT.SimpleTy != MVT::v8f16)
4375 return 0;
4376 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4377 return fastEmitInst_rr(ARM::VMINhq, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4378 }
4379 return 0;
4380}
4381
4382unsigned fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4383 if (RetVT.SimpleTy != MVT::v2f32)
4384 return 0;
4385 if ((Subtarget->hasNEON())) {
4386 return fastEmitInst_rr(ARM::VMINfd, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4387 }
4388 return 0;
4389}
4390
4391unsigned fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4392 if (RetVT.SimpleTy != MVT::v4f32)
4393 return 0;
4394 if ((Subtarget->hasNEON())) {
4395 return fastEmitInst_rr(ARM::VMINfq, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4396 }
4397 return 0;
4398}
4399
4400unsigned fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4401 switch (VT.SimpleTy) {
4402 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4403 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4404 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4405 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4406 default: return 0;
4407 }
4408}
4409
4410// FastEmit functions for ISD::FMINNUM.
4411
4412unsigned fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4413 if (RetVT.SimpleTy != MVT::f16)
4414 return 0;
4415 if ((Subtarget->hasFullFP16())) {
4416 return fastEmitInst_rr(ARM::VFP_VMINNMH, &ARM::HPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4417 }
4418 return 0;
4419}
4420
4421unsigned fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4422 if (RetVT.SimpleTy != MVT::f32)
4423 return 0;
4424 if ((Subtarget->hasFPARMv8Base())) {
4425 return fastEmitInst_rr(ARM::VFP_VMINNMS, &ARM::SPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4426 }
4427 return 0;
4428}
4429
4430unsigned fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4431 if (RetVT.SimpleTy != MVT::f64)
4432 return 0;
4433 if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
4434 return fastEmitInst_rr(ARM::VFP_VMINNMD, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4435 }
4436 return 0;
4437}
4438
4439unsigned fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4440 if (RetVT.SimpleTy != MVT::v4f16)
4441 return 0;
4442 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
4443 return fastEmitInst_rr(ARM::NEON_VMINNMNDh, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4444 }
4445 return 0;
4446}
4447
4448unsigned fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4449 if (RetVT.SimpleTy != MVT::v8f16)
4450 return 0;
4451 if ((Subtarget->hasMVEFloatOps())) {
4452 return fastEmitInst_rr(ARM::MVE_VMINNMf16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4453 }
4454 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
4455 return fastEmitInst_rr(ARM::NEON_VMINNMNQh, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4456 }
4457 return 0;
4458}
4459
4460unsigned fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4461 if (RetVT.SimpleTy != MVT::v2f32)
4462 return 0;
4463 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
4464 return fastEmitInst_rr(ARM::NEON_VMINNMNDf, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4465 }
4466 return 0;
4467}
4468
4469unsigned fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4470 if (RetVT.SimpleTy != MVT::v4f32)
4471 return 0;
4472 if ((Subtarget->hasMVEFloatOps())) {
4473 return fastEmitInst_rr(ARM::MVE_VMINNMf32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4474 }
4475 if ((Subtarget->hasNEON()) && (Subtarget->hasV8Ops())) {
4476 return fastEmitInst_rr(ARM::NEON_VMINNMNQf, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4477 }
4478 return 0;
4479}
4480
4481unsigned fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4482 switch (VT.SimpleTy) {
4483 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4484 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4485 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4486 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4487 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4488 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4489 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4490 default: return 0;
4491 }
4492}
4493
4494// FastEmit functions for ISD::FMUL.
4495
4496unsigned fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4497 if (RetVT.SimpleTy != MVT::f16)
4498 return 0;
4499 if ((Subtarget->hasFullFP16())) {
4500 return fastEmitInst_rr(ARM::VMULH, &ARM::HPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4501 }
4502 return 0;
4503}
4504
4505unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4506 if (RetVT.SimpleTy != MVT::f32)
4507 return 0;
4508 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
4509 return fastEmitInst_rr(ARM::VMULS, &ARM::SPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4510 }
4511 return 0;
4512}
4513
4514unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4515 if (RetVT.SimpleTy != MVT::f64)
4516 return 0;
4517 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
4518 return fastEmitInst_rr(ARM::VMULD, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4519 }
4520 return 0;
4521}
4522
4523unsigned fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4524 if (RetVT.SimpleTy != MVT::v4f16)
4525 return 0;
4526 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4527 return fastEmitInst_rr(ARM::VMULhd, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4528 }
4529 return 0;
4530}
4531
4532unsigned fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4533 if (RetVT.SimpleTy != MVT::v8f16)
4534 return 0;
4535 if ((Subtarget->hasMVEFloatOps())) {
4536 return fastEmitInst_rr(ARM::MVE_VMULf16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4537 }
4538 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4539 return fastEmitInst_rr(ARM::VMULhq, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4540 }
4541 return 0;
4542}
4543
4544unsigned fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4545 if (RetVT.SimpleTy != MVT::v2f32)
4546 return 0;
4547 if ((Subtarget->hasNEON())) {
4548 return fastEmitInst_rr(ARM::VMULfd, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4549 }
4550 return 0;
4551}
4552
4553unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4554 if (RetVT.SimpleTy != MVT::v4f32)
4555 return 0;
4556 if ((Subtarget->hasMVEFloatOps())) {
4557 return fastEmitInst_rr(ARM::MVE_VMULf32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4558 }
4559 if ((Subtarget->hasNEON())) {
4560 return fastEmitInst_rr(ARM::VMULfq, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4561 }
4562 return 0;
4563}
4564
4565unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4566 switch (VT.SimpleTy) {
4567 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4568 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4569 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4570 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4571 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4572 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4573 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4574 default: return 0;
4575 }
4576}
4577
4578// FastEmit functions for ISD::FSUB.
4579
4580unsigned fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4581 if (RetVT.SimpleTy != MVT::f16)
4582 return 0;
4583 if ((Subtarget->hasFullFP16())) {
4584 return fastEmitInst_rr(ARM::VSUBH, &ARM::HPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4585 }
4586 return 0;
4587}
4588
4589unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4590 if (RetVT.SimpleTy != MVT::f32)
4591 return 0;
4592 if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
4593 return fastEmitInst_rr(ARM::VSUBS, &ARM::SPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4594 }
4595 return 0;
4596}
4597
4598unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4599 if (RetVT.SimpleTy != MVT::f64)
4600 return 0;
4601 if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
4602 return fastEmitInst_rr(ARM::VSUBD, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4603 }
4604 return 0;
4605}
4606
4607unsigned fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4608 if (RetVT.SimpleTy != MVT::v4f16)
4609 return 0;
4610 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4611 return fastEmitInst_rr(ARM::VSUBhd, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4612 }
4613 return 0;
4614}
4615
4616unsigned fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4617 if (RetVT.SimpleTy != MVT::v8f16)
4618 return 0;
4619 if ((Subtarget->hasMVEFloatOps())) {
4620 return fastEmitInst_rr(ARM::MVE_VSUBf16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4621 }
4622 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
4623 return fastEmitInst_rr(ARM::VSUBhq, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4624 }
4625 return 0;
4626}
4627
4628unsigned fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4629 if (RetVT.SimpleTy != MVT::v2f32)
4630 return 0;
4631 if ((Subtarget->hasNEON())) {
4632 return fastEmitInst_rr(ARM::VSUBfd, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4633 }
4634 return 0;
4635}
4636
4637unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4638 if (RetVT.SimpleTy != MVT::v4f32)
4639 return 0;
4640 if ((Subtarget->hasMVEFloatOps())) {
4641 return fastEmitInst_rr(ARM::MVE_VSUBf32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4642 }
4643 if ((Subtarget->hasNEON())) {
4644 return fastEmitInst_rr(ARM::VSUBfq, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4645 }
4646 return 0;
4647}
4648
4649unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4650 switch (VT.SimpleTy) {
4651 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4652 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4653 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4654 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4655 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4656 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4657 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4658 default: return 0;
4659 }
4660}
4661
4662// FastEmit functions for ISD::MUL.
4663
4664unsigned fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4665 if (RetVT.SimpleTy != MVT::i32)
4666 return 0;
4667 if ((Subtarget->isThumb2())) {
4668 return fastEmitInst_rr(ARM::t2MUL, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4669 }
4670 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4671 return fastEmitInst_rr(ARM::tMUL, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4672 }
4673 if ((!Subtarget->isThumb()) && (!Subtarget->hasV6Ops()) && (Subtarget->useMulOps())) {
4674 return fastEmitInst_rr(ARM::MULv5, &ARM::GPRnopcRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4675 }
4676 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4677 return fastEmitInst_rr(ARM::MUL, &ARM::GPRnopcRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4678 }
4679 return 0;
4680}
4681
4682unsigned fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4683 if (RetVT.SimpleTy != MVT::v8i8)
4684 return 0;
4685 if ((Subtarget->hasNEON())) {
4686 return fastEmitInst_rr(ARM::VMULv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4687 }
4688 return 0;
4689}
4690
4691unsigned fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4692 if (RetVT.SimpleTy != MVT::v16i8)
4693 return 0;
4694 if ((Subtarget->hasMVEIntegerOps())) {
4695 return fastEmitInst_rr(ARM::MVE_VMULi8, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4696 }
4697 if ((Subtarget->hasNEON())) {
4698 return fastEmitInst_rr(ARM::VMULv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4699 }
4700 return 0;
4701}
4702
4703unsigned fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4704 if (RetVT.SimpleTy != MVT::v4i16)
4705 return 0;
4706 if ((Subtarget->hasNEON())) {
4707 return fastEmitInst_rr(ARM::VMULv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4708 }
4709 return 0;
4710}
4711
4712unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4713 if (RetVT.SimpleTy != MVT::v8i16)
4714 return 0;
4715 if ((Subtarget->hasMVEIntegerOps())) {
4716 return fastEmitInst_rr(ARM::MVE_VMULi16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4717 }
4718 if ((Subtarget->hasNEON())) {
4719 return fastEmitInst_rr(ARM::VMULv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4720 }
4721 return 0;
4722}
4723
4724unsigned fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4725 if (RetVT.SimpleTy != MVT::v2i32)
4726 return 0;
4727 if ((Subtarget->hasNEON())) {
4728 return fastEmitInst_rr(ARM::VMULv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4729 }
4730 return 0;
4731}
4732
4733unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4734 if (RetVT.SimpleTy != MVT::v4i32)
4735 return 0;
4736 if ((Subtarget->hasMVEIntegerOps())) {
4737 return fastEmitInst_rr(ARM::MVE_VMULi32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4738 }
4739 if ((Subtarget->hasNEON())) {
4740 return fastEmitInst_rr(ARM::VMULv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4741 }
4742 return 0;
4743}
4744
4745unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4746 switch (VT.SimpleTy) {
4747 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4748 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4749 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4750 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4751 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4752 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4753 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4754 default: return 0;
4755 }
4756}
4757
4758// FastEmit functions for ISD::MULHS.
4759
4760unsigned fastEmit_ISD_MULHS_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4761 if (RetVT.SimpleTy != MVT::i32)
4762 return 0;
4763 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4764 return fastEmitInst_rr(ARM::t2SMMUL, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4765 }
4766 if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
4767 return fastEmitInst_rr(ARM::SMMUL, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4768 }
4769 return 0;
4770}
4771
4772unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4773 switch (VT.SimpleTy) {
4774 case MVT::i32: return fastEmit_ISD_MULHS_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4775 default: return 0;
4776 }
4777}
4778
4779// FastEmit functions for ISD::OR.
4780
4781unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4782 if (RetVT.SimpleTy != MVT::i32)
4783 return 0;
4784 if ((Subtarget->isThumb2())) {
4785 return fastEmitInst_rr(ARM::t2ORRrr, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4786 }
4787 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4788 return fastEmitInst_rr(ARM::tORR, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4789 }
4790 if ((!Subtarget->isThumb())) {
4791 return fastEmitInst_rr(ARM::ORRrr, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4792 }
4793 return 0;
4794}
4795
4796unsigned fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4797 if (RetVT.SimpleTy != MVT::v16i8)
4798 return 0;
4799 if ((Subtarget->hasMVEIntegerOps())) {
4800 return fastEmitInst_rr(ARM::MVE_VORR, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4801 }
4802 return 0;
4803}
4804
4805unsigned fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4806 if (RetVT.SimpleTy != MVT::v8i16)
4807 return 0;
4808 if ((Subtarget->hasMVEIntegerOps())) {
4809 return fastEmitInst_rr(ARM::MVE_VORR, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4810 }
4811 return 0;
4812}
4813
4814unsigned fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4815 if (RetVT.SimpleTy != MVT::v2i32)
4816 return 0;
4817 if ((Subtarget->hasNEON())) {
4818 return fastEmitInst_rr(ARM::VORRd, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4819 }
4820 return 0;
4821}
4822
4823unsigned fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4824 if (RetVT.SimpleTy != MVT::v4i32)
4825 return 0;
4826 if ((Subtarget->hasMVEIntegerOps())) {
4827 return fastEmitInst_rr(ARM::MVE_VORR, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4828 }
4829 if ((Subtarget->hasNEON())) {
4830 return fastEmitInst_rr(ARM::VORRq, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4831 }
4832 return 0;
4833}
4834
4835unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4836 if (RetVT.SimpleTy != MVT::v2i64)
4837 return 0;
4838 if ((Subtarget->hasMVEIntegerOps())) {
4839 return fastEmitInst_rr(ARM::MVE_VORR, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4840 }
4841 return 0;
4842}
4843
4844unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4845 switch (VT.SimpleTy) {
4846 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4847 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4848 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4849 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4850 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4851 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4852 default: return 0;
4853 }
4854}
4855
4856// FastEmit functions for ISD::ROTR.
4857
4858unsigned fastEmit_ISD_ROTR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4859 if (RetVT.SimpleTy != MVT::i32)
4860 return 0;
4861 if ((Subtarget->isThumb2())) {
4862 return fastEmitInst_rr(ARM::t2RORrr, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4863 }
4864 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
4865 return fastEmitInst_rr(ARM::tROR, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4866 }
4867 return 0;
4868}
4869
4870unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4871 switch (VT.SimpleTy) {
4872 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4873 default: return 0;
4874 }
4875}
4876
4877// FastEmit functions for ISD::SADDSAT.
4878
4879unsigned fastEmit_ISD_SADDSAT_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4880 if (RetVT.SimpleTy != MVT::i32)
4881 return 0;
4882 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
4883 return fastEmitInst_rr(ARM::t2QADD, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4884 }
4885 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
4886 return fastEmitInst_rr(ARM::QADD, &ARM::GPRnopcRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4887 }
4888 return 0;
4889}
4890
4891unsigned fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4892 if (RetVT.SimpleTy != MVT::v8i8)
4893 return 0;
4894 if ((Subtarget->hasNEON())) {
4895 return fastEmitInst_rr(ARM::VQADDsv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4896 }
4897 return 0;
4898}
4899
4900unsigned fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4901 if (RetVT.SimpleTy != MVT::v16i8)
4902 return 0;
4903 if ((Subtarget->hasMVEIntegerOps())) {
4904 return fastEmitInst_rr(ARM::MVE_VQADDs8, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4905 }
4906 if ((Subtarget->hasNEON())) {
4907 return fastEmitInst_rr(ARM::VQADDsv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4908 }
4909 return 0;
4910}
4911
4912unsigned fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4913 if (RetVT.SimpleTy != MVT::v4i16)
4914 return 0;
4915 if ((Subtarget->hasNEON())) {
4916 return fastEmitInst_rr(ARM::VQADDsv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4917 }
4918 return 0;
4919}
4920
4921unsigned fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4922 if (RetVT.SimpleTy != MVT::v8i16)
4923 return 0;
4924 if ((Subtarget->hasMVEIntegerOps())) {
4925 return fastEmitInst_rr(ARM::MVE_VQADDs16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4926 }
4927 if ((Subtarget->hasNEON())) {
4928 return fastEmitInst_rr(ARM::VQADDsv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4929 }
4930 return 0;
4931}
4932
4933unsigned fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4934 if (RetVT.SimpleTy != MVT::v2i32)
4935 return 0;
4936 if ((Subtarget->hasNEON())) {
4937 return fastEmitInst_rr(ARM::VQADDsv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4938 }
4939 return 0;
4940}
4941
4942unsigned fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4943 if (RetVT.SimpleTy != MVT::v4i32)
4944 return 0;
4945 if ((Subtarget->hasMVEIntegerOps())) {
4946 return fastEmitInst_rr(ARM::MVE_VQADDs32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4947 }
4948 if ((Subtarget->hasNEON())) {
4949 return fastEmitInst_rr(ARM::VQADDsv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4950 }
4951 return 0;
4952}
4953
4954unsigned fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4955 if (RetVT.SimpleTy != MVT::v1i64)
4956 return 0;
4957 if ((Subtarget->hasNEON())) {
4958 return fastEmitInst_rr(ARM::VQADDsv1i64, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4959 }
4960 return 0;
4961}
4962
4963unsigned fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4964 if (RetVT.SimpleTy != MVT::v2i64)
4965 return 0;
4966 if ((Subtarget->hasNEON())) {
4967 return fastEmitInst_rr(ARM::VQADDsv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4968 }
4969 return 0;
4970}
4971
4972unsigned fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4973 switch (VT.SimpleTy) {
4974 case MVT::i32: return fastEmit_ISD_SADDSAT_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4975 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4976 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4977 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4978 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4979 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4980 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4981 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4982 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
4983 default: return 0;
4984 }
4985}
4986
4987// FastEmit functions for ISD::SDIV.
4988
4989unsigned fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
4990 if (RetVT.SimpleTy != MVT::i32)
4991 return 0;
4992 if ((Subtarget->hasDivideInThumbMode()) && (Subtarget->hasV8MBaselineOps()) && (Subtarget->isThumb())) {
4993 return fastEmitInst_rr(ARM::t2SDIV, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4994 }
4995 if ((Subtarget->hasDivideInARMMode()) && (!Subtarget->isThumb())) {
4996 return fastEmitInst_rr(ARM::SDIV, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
4997 }
4998 return 0;
4999}
5000
5001unsigned fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5002 switch (VT.SimpleTy) {
5003 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5004 default: return 0;
5005 }
5006}
5007
5008// FastEmit functions for ISD::SHL.
5009
5010unsigned fastEmit_ISD_SHL_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5011 if (RetVT.SimpleTy != MVT::i32)
5012 return 0;
5013 if ((Subtarget->isThumb2())) {
5014 return fastEmitInst_rr(ARM::t2LSLrr, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5015 }
5016 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5017 return fastEmitInst_rr(ARM::tLSLrr, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5018 }
5019 return 0;
5020}
5021
5022unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5023 switch (VT.SimpleTy) {
5024 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5025 default: return 0;
5026 }
5027}
5028
5029// FastEmit functions for ISD::SMAX.
5030
5031unsigned fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5032 if (RetVT.SimpleTy != MVT::v8i8)
5033 return 0;
5034 if ((Subtarget->hasNEON())) {
5035 return fastEmitInst_rr(ARM::VMAXsv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5036 }
5037 return 0;
5038}
5039
5040unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5041 if (RetVT.SimpleTy != MVT::v16i8)
5042 return 0;
5043 if ((Subtarget->hasMVEIntegerOps())) {
5044 return fastEmitInst_rr(ARM::MVE_VMAXs8, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5045 }
5046 if ((Subtarget->hasNEON())) {
5047 return fastEmitInst_rr(ARM::VMAXsv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5048 }
5049 return 0;
5050}
5051
5052unsigned fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5053 if (RetVT.SimpleTy != MVT::v4i16)
5054 return 0;
5055 if ((Subtarget->hasNEON())) {
5056 return fastEmitInst_rr(ARM::VMAXsv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5057 }
5058 return 0;
5059}
5060
5061unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5062 if (RetVT.SimpleTy != MVT::v8i16)
5063 return 0;
5064 if ((Subtarget->hasMVEIntegerOps())) {
5065 return fastEmitInst_rr(ARM::MVE_VMAXs16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5066 }
5067 if ((Subtarget->hasNEON())) {
5068 return fastEmitInst_rr(ARM::VMAXsv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5069 }
5070 return 0;
5071}
5072
5073unsigned fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5074 if (RetVT.SimpleTy != MVT::v2i32)
5075 return 0;
5076 if ((Subtarget->hasNEON())) {
5077 return fastEmitInst_rr(ARM::VMAXsv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5078 }
5079 return 0;
5080}
5081
5082unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5083 if (RetVT.SimpleTy != MVT::v4i32)
5084 return 0;
5085 if ((Subtarget->hasMVEIntegerOps())) {
5086 return fastEmitInst_rr(ARM::MVE_VMAXs32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5087 }
5088 if ((Subtarget->hasNEON())) {
5089 return fastEmitInst_rr(ARM::VMAXsv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5090 }
5091 return 0;
5092}
5093
5094unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5095 switch (VT.SimpleTy) {
5096 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5097 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5098 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5099 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5100 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5101 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5102 default: return 0;
5103 }
5104}
5105
5106// FastEmit functions for ISD::SMIN.
5107
5108unsigned fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5109 if (RetVT.SimpleTy != MVT::v8i8)
5110 return 0;
5111 if ((Subtarget->hasNEON())) {
5112 return fastEmitInst_rr(ARM::VMINsv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5113 }
5114 return 0;
5115}
5116
5117unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5118 if (RetVT.SimpleTy != MVT::v16i8)
5119 return 0;
5120 if ((Subtarget->hasMVEIntegerOps())) {
5121 return fastEmitInst_rr(ARM::MVE_VMINs8, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5122 }
5123 if ((Subtarget->hasNEON())) {
5124 return fastEmitInst_rr(ARM::VMINsv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5125 }
5126 return 0;
5127}
5128
5129unsigned fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5130 if (RetVT.SimpleTy != MVT::v4i16)
5131 return 0;
5132 if ((Subtarget->hasNEON())) {
5133 return fastEmitInst_rr(ARM::VMINsv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5134 }
5135 return 0;
5136}
5137
5138unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5139 if (RetVT.SimpleTy != MVT::v8i16)
5140 return 0;
5141 if ((Subtarget->hasMVEIntegerOps())) {
5142 return fastEmitInst_rr(ARM::MVE_VMINs16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5143 }
5144 if ((Subtarget->hasNEON())) {
5145 return fastEmitInst_rr(ARM::VMINsv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5146 }
5147 return 0;
5148}
5149
5150unsigned fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5151 if (RetVT.SimpleTy != MVT::v2i32)
5152 return 0;
5153 if ((Subtarget->hasNEON())) {
5154 return fastEmitInst_rr(ARM::VMINsv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5155 }
5156 return 0;
5157}
5158
5159unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5160 if (RetVT.SimpleTy != MVT::v4i32)
5161 return 0;
5162 if ((Subtarget->hasMVEIntegerOps())) {
5163 return fastEmitInst_rr(ARM::MVE_VMINs32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5164 }
5165 if ((Subtarget->hasNEON())) {
5166 return fastEmitInst_rr(ARM::VMINsv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5167 }
5168 return 0;
5169}
5170
5171unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5172 switch (VT.SimpleTy) {
5173 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5174 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5175 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5176 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5177 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5178 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5179 default: return 0;
5180 }
5181}
5182
5183// FastEmit functions for ISD::SRA.
5184
5185unsigned fastEmit_ISD_SRA_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5186 if (RetVT.SimpleTy != MVT::i32)
5187 return 0;
5188 if ((Subtarget->isThumb2())) {
5189 return fastEmitInst_rr(ARM::t2ASRrr, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5190 }
5191 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5192 return fastEmitInst_rr(ARM::tASRrr, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5193 }
5194 return 0;
5195}
5196
5197unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5198 switch (VT.SimpleTy) {
5199 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5200 default: return 0;
5201 }
5202}
5203
5204// FastEmit functions for ISD::SRL.
5205
5206unsigned fastEmit_ISD_SRL_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5207 if (RetVT.SimpleTy != MVT::i32)
5208 return 0;
5209 if ((Subtarget->isThumb2())) {
5210 return fastEmitInst_rr(ARM::t2LSRrr, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5211 }
5212 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5213 return fastEmitInst_rr(ARM::tLSRrr, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5214 }
5215 return 0;
5216}
5217
5218unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5219 switch (VT.SimpleTy) {
5220 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5221 default: return 0;
5222 }
5223}
5224
5225// FastEmit functions for ISD::SSUBSAT.
5226
5227unsigned fastEmit_ISD_SSUBSAT_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5228 if (RetVT.SimpleTy != MVT::i32)
5229 return 0;
5230 if ((Subtarget->hasDSP()) && (Subtarget->isThumb2())) {
5231 return fastEmitInst_rr(ARM::t2QSUB, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5232 }
5233 if ((Subtarget->hasV5TEOps()) && (!Subtarget->isThumb())) {
5234 return fastEmitInst_rr(ARM::QSUB, &ARM::GPRnopcRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5235 }
5236 return 0;
5237}
5238
5239unsigned fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5240 if (RetVT.SimpleTy != MVT::v8i8)
5241 return 0;
5242 if ((Subtarget->hasNEON())) {
5243 return fastEmitInst_rr(ARM::VQSUBsv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5244 }
5245 return 0;
5246}
5247
5248unsigned fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5249 if (RetVT.SimpleTy != MVT::v16i8)
5250 return 0;
5251 if ((Subtarget->hasMVEIntegerOps())) {
5252 return fastEmitInst_rr(ARM::MVE_VQSUBs8, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5253 }
5254 if ((Subtarget->hasNEON())) {
5255 return fastEmitInst_rr(ARM::VQSUBsv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5256 }
5257 return 0;
5258}
5259
5260unsigned fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5261 if (RetVT.SimpleTy != MVT::v4i16)
5262 return 0;
5263 if ((Subtarget->hasNEON())) {
5264 return fastEmitInst_rr(ARM::VQSUBsv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5265 }
5266 return 0;
5267}
5268
5269unsigned fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5270 if (RetVT.SimpleTy != MVT::v8i16)
5271 return 0;
5272 if ((Subtarget->hasMVEIntegerOps())) {
5273 return fastEmitInst_rr(ARM::MVE_VQSUBs16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5274 }
5275 if ((Subtarget->hasNEON())) {
5276 return fastEmitInst_rr(ARM::VQSUBsv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5277 }
5278 return 0;
5279}
5280
5281unsigned fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5282 if (RetVT.SimpleTy != MVT::v2i32)
5283 return 0;
5284 if ((Subtarget->hasNEON())) {
5285 return fastEmitInst_rr(ARM::VQSUBsv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5286 }
5287 return 0;
5288}
5289
5290unsigned fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5291 if (RetVT.SimpleTy != MVT::v4i32)
5292 return 0;
5293 if ((Subtarget->hasMVEIntegerOps())) {
5294 return fastEmitInst_rr(ARM::MVE_VQSUBs32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5295 }
5296 if ((Subtarget->hasNEON())) {
5297 return fastEmitInst_rr(ARM::VQSUBsv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5298 }
5299 return 0;
5300}
5301
5302unsigned fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5303 if (RetVT.SimpleTy != MVT::v1i64)
5304 return 0;
5305 if ((Subtarget->hasNEON())) {
5306 return fastEmitInst_rr(ARM::VQSUBsv1i64, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5307 }
5308 return 0;
5309}
5310
5311unsigned fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5312 if (RetVT.SimpleTy != MVT::v2i64)
5313 return 0;
5314 if ((Subtarget->hasNEON())) {
5315 return fastEmitInst_rr(ARM::VQSUBsv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5316 }
5317 return 0;
5318}
5319
5320unsigned fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5321 switch (VT.SimpleTy) {
5322 case MVT::i32: return fastEmit_ISD_SSUBSAT_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5323 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5324 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5325 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5326 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5327 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5328 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5329 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5330 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5331 default: return 0;
5332 }
5333}
5334
5335// FastEmit functions for ISD::SUB.
5336
5337unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5338 if (RetVT.SimpleTy != MVT::i32)
5339 return 0;
5340 if ((Subtarget->isThumb2())) {
5341 return fastEmitInst_rr(ARM::t2SUBrr, &ARM::GPRnopcRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5342 }
5343 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5344 return fastEmitInst_rr(ARM::tSUBrr, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5345 }
5346 if ((!Subtarget->isThumb())) {
5347 return fastEmitInst_rr(ARM::SUBrr, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5348 }
5349 return 0;
5350}
5351
5352unsigned fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5353 if (RetVT.SimpleTy != MVT::v8i8)
5354 return 0;
5355 if ((Subtarget->hasNEON())) {
5356 return fastEmitInst_rr(ARM::VSUBv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5357 }
5358 return 0;
5359}
5360
5361unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5362 if (RetVT.SimpleTy != MVT::v16i8)
5363 return 0;
5364 if ((Subtarget->hasMVEIntegerOps())) {
5365 return fastEmitInst_rr(ARM::MVE_VSUBi8, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5366 }
5367 if ((Subtarget->hasNEON())) {
5368 return fastEmitInst_rr(ARM::VSUBv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5369 }
5370 return 0;
5371}
5372
5373unsigned fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5374 if (RetVT.SimpleTy != MVT::v4i16)
5375 return 0;
5376 if ((Subtarget->hasNEON())) {
5377 return fastEmitInst_rr(ARM::VSUBv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5378 }
5379 return 0;
5380}
5381
5382unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5383 if (RetVT.SimpleTy != MVT::v8i16)
5384 return 0;
5385 if ((Subtarget->hasMVEIntegerOps())) {
5386 return fastEmitInst_rr(ARM::MVE_VSUBi16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5387 }
5388 if ((Subtarget->hasNEON())) {
5389 return fastEmitInst_rr(ARM::VSUBv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5390 }
5391 return 0;
5392}
5393
5394unsigned fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5395 if (RetVT.SimpleTy != MVT::v2i32)
5396 return 0;
5397 if ((Subtarget->hasNEON())) {
5398 return fastEmitInst_rr(ARM::VSUBv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5399 }
5400 return 0;
5401}
5402
5403unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5404 if (RetVT.SimpleTy != MVT::v4i32)
5405 return 0;
5406 if ((Subtarget->hasMVEIntegerOps())) {
5407 return fastEmitInst_rr(ARM::MVE_VSUBi32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5408 }
5409 if ((Subtarget->hasNEON())) {
5410 return fastEmitInst_rr(ARM::VSUBv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5411 }
5412 return 0;
5413}
5414
5415unsigned fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5416 if (RetVT.SimpleTy != MVT::v1i64)
5417 return 0;
5418 if ((Subtarget->hasNEON())) {
5419 return fastEmitInst_rr(ARM::VSUBv1i64, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5420 }
5421 return 0;
5422}
5423
5424unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5425 if (RetVT.SimpleTy != MVT::v2i64)
5426 return 0;
5427 if ((Subtarget->hasNEON())) {
5428 return fastEmitInst_rr(ARM::VSUBv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5429 }
5430 return 0;
5431}
5432
5433unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5434 switch (VT.SimpleTy) {
5435 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5436 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5437 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5438 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5439 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5440 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5441 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5442 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5443 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5444 default: return 0;
5445 }
5446}
5447
5448// FastEmit functions for ISD::UADDSAT.
5449
5450unsigned fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5451 if (RetVT.SimpleTy != MVT::v8i8)
5452 return 0;
5453 if ((Subtarget->hasNEON())) {
5454 return fastEmitInst_rr(ARM::VQADDuv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5455 }
5456 return 0;
5457}
5458
5459unsigned fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5460 if (RetVT.SimpleTy != MVT::v16i8)
5461 return 0;
5462 if ((Subtarget->hasMVEIntegerOps())) {
5463 return fastEmitInst_rr(ARM::MVE_VQADDu8, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5464 }
5465 if ((Subtarget->hasNEON())) {
5466 return fastEmitInst_rr(ARM::VQADDuv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5467 }
5468 return 0;
5469}
5470
5471unsigned fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5472 if (RetVT.SimpleTy != MVT::v4i16)
5473 return 0;
5474 if ((Subtarget->hasNEON())) {
5475 return fastEmitInst_rr(ARM::VQADDuv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5476 }
5477 return 0;
5478}
5479
5480unsigned fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5481 if (RetVT.SimpleTy != MVT::v8i16)
5482 return 0;
5483 if ((Subtarget->hasMVEIntegerOps())) {
5484 return fastEmitInst_rr(ARM::MVE_VQADDu16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5485 }
5486 if ((Subtarget->hasNEON())) {
5487 return fastEmitInst_rr(ARM::VQADDuv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5488 }
5489 return 0;
5490}
5491
5492unsigned fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5493 if (RetVT.SimpleTy != MVT::v2i32)
5494 return 0;
5495 if ((Subtarget->hasNEON())) {
5496 return fastEmitInst_rr(ARM::VQADDuv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5497 }
5498 return 0;
5499}
5500
5501unsigned fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5502 if (RetVT.SimpleTy != MVT::v4i32)
5503 return 0;
5504 if ((Subtarget->hasMVEIntegerOps())) {
5505 return fastEmitInst_rr(ARM::MVE_VQADDu32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5506 }
5507 if ((Subtarget->hasNEON())) {
5508 return fastEmitInst_rr(ARM::VQADDuv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5509 }
5510 return 0;
5511}
5512
5513unsigned fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5514 if (RetVT.SimpleTy != MVT::v1i64)
5515 return 0;
5516 if ((Subtarget->hasNEON())) {
5517 return fastEmitInst_rr(ARM::VQADDuv1i64, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5518 }
5519 return 0;
5520}
5521
5522unsigned fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5523 if (RetVT.SimpleTy != MVT::v2i64)
5524 return 0;
5525 if ((Subtarget->hasNEON())) {
5526 return fastEmitInst_rr(ARM::VQADDuv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5527 }
5528 return 0;
5529}
5530
5531unsigned fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5532 switch (VT.SimpleTy) {
5533 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5534 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5535 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5536 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5537 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5538 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5539 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5540 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5541 default: return 0;
5542 }
5543}
5544
5545// FastEmit functions for ISD::UDIV.
5546
5547unsigned fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5548 if (RetVT.SimpleTy != MVT::i32)
5549 return 0;
5550 if ((Subtarget->hasDivideInThumbMode()) && (Subtarget->hasV8MBaselineOps()) && (Subtarget->isThumb())) {
5551 return fastEmitInst_rr(ARM::t2UDIV, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5552 }
5553 if ((Subtarget->hasDivideInARMMode()) && (!Subtarget->isThumb())) {
5554 return fastEmitInst_rr(ARM::UDIV, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5555 }
5556 return 0;
5557}
5558
5559unsigned fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5560 switch (VT.SimpleTy) {
5561 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5562 default: return 0;
5563 }
5564}
5565
5566// FastEmit functions for ISD::UMAX.
5567
5568unsigned fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5569 if (RetVT.SimpleTy != MVT::v8i8)
5570 return 0;
5571 if ((Subtarget->hasNEON())) {
5572 return fastEmitInst_rr(ARM::VMAXuv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5573 }
5574 return 0;
5575}
5576
5577unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5578 if (RetVT.SimpleTy != MVT::v16i8)
5579 return 0;
5580 if ((Subtarget->hasMVEIntegerOps())) {
5581 return fastEmitInst_rr(ARM::MVE_VMAXu8, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5582 }
5583 if ((Subtarget->hasNEON())) {
5584 return fastEmitInst_rr(ARM::VMAXuv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5585 }
5586 return 0;
5587}
5588
5589unsigned fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5590 if (RetVT.SimpleTy != MVT::v4i16)
5591 return 0;
5592 if ((Subtarget->hasNEON())) {
5593 return fastEmitInst_rr(ARM::VMAXuv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5594 }
5595 return 0;
5596}
5597
5598unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5599 if (RetVT.SimpleTy != MVT::v8i16)
5600 return 0;
5601 if ((Subtarget->hasMVEIntegerOps())) {
5602 return fastEmitInst_rr(ARM::MVE_VMAXu16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5603 }
5604 if ((Subtarget->hasNEON())) {
5605 return fastEmitInst_rr(ARM::VMAXuv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5606 }
5607 return 0;
5608}
5609
5610unsigned fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5611 if (RetVT.SimpleTy != MVT::v2i32)
5612 return 0;
5613 if ((Subtarget->hasNEON())) {
5614 return fastEmitInst_rr(ARM::VMAXuv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5615 }
5616 return 0;
5617}
5618
5619unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5620 if (RetVT.SimpleTy != MVT::v4i32)
5621 return 0;
5622 if ((Subtarget->hasMVEIntegerOps())) {
5623 return fastEmitInst_rr(ARM::MVE_VMAXu32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5624 }
5625 if ((Subtarget->hasNEON())) {
5626 return fastEmitInst_rr(ARM::VMAXuv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5627 }
5628 return 0;
5629}
5630
5631unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5632 switch (VT.SimpleTy) {
5633 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5634 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5635 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5636 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5637 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5638 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5639 default: return 0;
5640 }
5641}
5642
5643// FastEmit functions for ISD::UMIN.
5644
5645unsigned fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5646 if (RetVT.SimpleTy != MVT::v8i8)
5647 return 0;
5648 if ((Subtarget->hasNEON())) {
5649 return fastEmitInst_rr(ARM::VMINuv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5650 }
5651 return 0;
5652}
5653
5654unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5655 if (RetVT.SimpleTy != MVT::v16i8)
5656 return 0;
5657 if ((Subtarget->hasMVEIntegerOps())) {
5658 return fastEmitInst_rr(ARM::MVE_VMINu8, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5659 }
5660 if ((Subtarget->hasNEON())) {
5661 return fastEmitInst_rr(ARM::VMINuv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5662 }
5663 return 0;
5664}
5665
5666unsigned fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5667 if (RetVT.SimpleTy != MVT::v4i16)
5668 return 0;
5669 if ((Subtarget->hasNEON())) {
5670 return fastEmitInst_rr(ARM::VMINuv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5671 }
5672 return 0;
5673}
5674
5675unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5676 if (RetVT.SimpleTy != MVT::v8i16)
5677 return 0;
5678 if ((Subtarget->hasMVEIntegerOps())) {
5679 return fastEmitInst_rr(ARM::MVE_VMINu16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5680 }
5681 if ((Subtarget->hasNEON())) {
5682 return fastEmitInst_rr(ARM::VMINuv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5683 }
5684 return 0;
5685}
5686
5687unsigned fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5688 if (RetVT.SimpleTy != MVT::v2i32)
5689 return 0;
5690 if ((Subtarget->hasNEON())) {
5691 return fastEmitInst_rr(ARM::VMINuv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5692 }
5693 return 0;
5694}
5695
5696unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5697 if (RetVT.SimpleTy != MVT::v4i32)
5698 return 0;
5699 if ((Subtarget->hasMVEIntegerOps())) {
5700 return fastEmitInst_rr(ARM::MVE_VMINu32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5701 }
5702 if ((Subtarget->hasNEON())) {
5703 return fastEmitInst_rr(ARM::VMINuv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5704 }
5705 return 0;
5706}
5707
5708unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5709 switch (VT.SimpleTy) {
5710 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5711 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5712 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5713 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5714 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5715 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5716 default: return 0;
5717 }
5718}
5719
5720// FastEmit functions for ISD::USUBSAT.
5721
5722unsigned fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5723 if (RetVT.SimpleTy != MVT::v8i8)
5724 return 0;
5725 if ((Subtarget->hasNEON())) {
5726 return fastEmitInst_rr(ARM::VQSUBuv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5727 }
5728 return 0;
5729}
5730
5731unsigned fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5732 if (RetVT.SimpleTy != MVT::v16i8)
5733 return 0;
5734 if ((Subtarget->hasMVEIntegerOps())) {
5735 return fastEmitInst_rr(ARM::MVE_VQSUBu8, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5736 }
5737 if ((Subtarget->hasNEON())) {
5738 return fastEmitInst_rr(ARM::VQSUBuv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5739 }
5740 return 0;
5741}
5742
5743unsigned fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5744 if (RetVT.SimpleTy != MVT::v4i16)
5745 return 0;
5746 if ((Subtarget->hasNEON())) {
5747 return fastEmitInst_rr(ARM::VQSUBuv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5748 }
5749 return 0;
5750}
5751
5752unsigned fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5753 if (RetVT.SimpleTy != MVT::v8i16)
5754 return 0;
5755 if ((Subtarget->hasMVEIntegerOps())) {
5756 return fastEmitInst_rr(ARM::MVE_VQSUBu16, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5757 }
5758 if ((Subtarget->hasNEON())) {
5759 return fastEmitInst_rr(ARM::VQSUBuv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5760 }
5761 return 0;
5762}
5763
5764unsigned fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5765 if (RetVT.SimpleTy != MVT::v2i32)
5766 return 0;
5767 if ((Subtarget->hasNEON())) {
5768 return fastEmitInst_rr(ARM::VQSUBuv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5769 }
5770 return 0;
5771}
5772
5773unsigned fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5774 if (RetVT.SimpleTy != MVT::v4i32)
5775 return 0;
5776 if ((Subtarget->hasMVEIntegerOps())) {
5777 return fastEmitInst_rr(ARM::MVE_VQSUBu32, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5778 }
5779 if ((Subtarget->hasNEON())) {
5780 return fastEmitInst_rr(ARM::VQSUBuv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5781 }
5782 return 0;
5783}
5784
5785unsigned fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5786 if (RetVT.SimpleTy != MVT::v1i64)
5787 return 0;
5788 if ((Subtarget->hasNEON())) {
5789 return fastEmitInst_rr(ARM::VQSUBuv1i64, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5790 }
5791 return 0;
5792}
5793
5794unsigned fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5795 if (RetVT.SimpleTy != MVT::v2i64)
5796 return 0;
5797 if ((Subtarget->hasNEON())) {
5798 return fastEmitInst_rr(ARM::VQSUBuv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5799 }
5800 return 0;
5801}
5802
5803unsigned fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5804 switch (VT.SimpleTy) {
5805 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5806 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5807 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5808 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5809 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5810 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5811 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5812 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5813 default: return 0;
5814 }
5815}
5816
5817// FastEmit functions for ISD::XOR.
5818
5819unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5820 if (RetVT.SimpleTy != MVT::i32)
5821 return 0;
5822 if ((Subtarget->isThumb2())) {
5823 return fastEmitInst_rr(ARM::t2EORrr, &ARM::rGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5824 }
5825 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
5826 return fastEmitInst_rr(ARM::tEOR, &ARM::tGPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5827 }
5828 if ((!Subtarget->isThumb())) {
5829 return fastEmitInst_rr(ARM::EORrr, &ARM::GPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5830 }
5831 return 0;
5832}
5833
5834unsigned fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5835 if (RetVT.SimpleTy != MVT::v16i8)
5836 return 0;
5837 if ((Subtarget->hasMVEIntegerOps())) {
5838 return fastEmitInst_rr(ARM::MVE_VEOR, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5839 }
5840 return 0;
5841}
5842
5843unsigned fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5844 if (RetVT.SimpleTy != MVT::v8i16)
5845 return 0;
5846 if ((Subtarget->hasMVEIntegerOps())) {
5847 return fastEmitInst_rr(ARM::MVE_VEOR, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5848 }
5849 return 0;
5850}
5851
5852unsigned fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5853 if (RetVT.SimpleTy != MVT::v2i32)
5854 return 0;
5855 if ((Subtarget->hasNEON())) {
5856 return fastEmitInst_rr(ARM::VEORd, &ARM::DPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5857 }
5858 return 0;
5859}
5860
5861unsigned fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5862 if (RetVT.SimpleTy != MVT::v4i32)
5863 return 0;
5864 if ((Subtarget->hasMVEIntegerOps())) {
5865 return fastEmitInst_rr(ARM::MVE_VEOR, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5866 }
5867 if ((Subtarget->hasNEON())) {
5868 return fastEmitInst_rr(ARM::VEORq, &ARM::QPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5869 }
5870 return 0;
5871}
5872
5873unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5874 if (RetVT.SimpleTy != MVT::v2i64)
5875 return 0;
5876 if ((Subtarget->hasMVEIntegerOps())) {
5877 return fastEmitInst_rr(ARM::MVE_VEOR, &ARM::MQPRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
5878 }
5879 return 0;
5880}
5881
5882unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
5883 switch (VT.SimpleTy) {
5884 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5885 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5886 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5887 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5888 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5889 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5890 default: return 0;
5891 }
5892}
5893
5894// Top-level FastEmit function.
5895
5896unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) override {
5897 switch (Opcode) {
5898 case ARMISD::CMP: return fastEmit_ARMISD_CMP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5899 case ARMISD::CMPFP: return fastEmit_ARMISD_CMPFP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5900 case ARMISD::CMPFPE: return fastEmit_ARMISD_CMPFPE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5901 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5902 case ARMISD::EH_SJLJ_LONGJMP: return fastEmit_ARMISD_EH_SJLJ_LONGJMP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5903 case ARMISD::EH_SJLJ_SETJMP: return fastEmit_ARMISD_EH_SJLJ_SETJMP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5904 case ARMISD::QADD16b: return fastEmit_ARMISD_QADD16b_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5905 case ARMISD::QADD8b: return fastEmit_ARMISD_QADD8b_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5906 case ARMISD::QSUB16b: return fastEmit_ARMISD_QSUB16b_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5907 case ARMISD::QSUB8b: return fastEmit_ARMISD_QSUB8b_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5908 case ARMISD::SMULWB: return fastEmit_ARMISD_SMULWB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5909 case ARMISD::SMULWT: return fastEmit_ARMISD_SMULWT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5910 case ARMISD::SUBS: return fastEmit_ARMISD_SUBS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5911 case ARMISD::VMLAVs: return fastEmit_ARMISD_VMLAVs_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5912 case ARMISD::VMLAVu: return fastEmit_ARMISD_VMLAVu_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5913 case ARMISD::VMOVDRR: return fastEmit_ARMISD_VMOVDRR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5914 case ARMISD::VMULLs: return fastEmit_ARMISD_VMULLs_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5915 case ARMISD::VMULLu: return fastEmit_ARMISD_VMULLu_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5916 case ARMISD::VQDMULH: return fastEmit_ARMISD_VQDMULH_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5917 case ARMISD::VSHLs: return fastEmit_ARMISD_VSHLs_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5918 case ARMISD::VSHLu: return fastEmit_ARMISD_VSHLu_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5919 case ARMISD::VTBL1: return fastEmit_ARMISD_VTBL1_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5920 case ARMISD::VTST: return fastEmit_ARMISD_VTST_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5921 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5922 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5923 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5924 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5925 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5926 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5927 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5928 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5929 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5930 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5931 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5932 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5933 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5934 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5935 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5936 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5937 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5938 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5939 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5940 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5941 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5942 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5943 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5944 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5945 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5946 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5947 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5948 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5949 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
5950 default: return 0;
5951 }
5952}
5953
5954// FastEmit functions for ARMISD::PIC_ADD.
5955
5956unsigned fastEmit_ARMISD_PIC_ADD_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
5957 if (RetVT.SimpleTy != MVT::i32)
5958 return 0;
5959 if ((Subtarget->isThumb())) {
5960 return fastEmitInst_ri(ARM::tPICADD, &ARM::GPRRegClass, Op0, Op0IsKill, imm1);
5961 }
5962 if ((!Subtarget->isThumb())) {
5963 return fastEmitInst_ri(ARM::PICADD, &ARM::GPRRegClass, Op0, Op0IsKill, imm1);
5964 }
5965 return 0;
5966}
5967
5968unsigned fastEmit_ARMISD_PIC_ADD_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
5969 switch (VT.SimpleTy) {
5970 case MVT::i32: return fastEmit_ARMISD_PIC_ADD_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
5971 default: return 0;
5972 }
5973}
5974
5975// FastEmit functions for ARMISD::VDUPLANE.
5976
5977unsigned fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
5978 if (RetVT.SimpleTy != MVT::v8i8)
5979 return 0;
5980 if ((Subtarget->hasNEON())) {
5981 return fastEmitInst_ri(ARM::VDUPLN8d, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
5982 }
5983 return 0;
5984}
5985
5986unsigned fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
5987 if (RetVT.SimpleTy != MVT::v4i16)
5988 return 0;
5989 if ((Subtarget->hasNEON())) {
5990 return fastEmitInst_ri(ARM::VDUPLN16d, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
5991 }
5992 return 0;
5993}
5994
5995unsigned fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
5996 if (RetVT.SimpleTy != MVT::v2i32)
5997 return 0;
5998 if ((Subtarget->hasNEON())) {
5999 return fastEmitInst_ri(ARM::VDUPLN32d, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6000 }
6001 return 0;
6002}
6003
6004unsigned fastEmit_ARMISD_VDUPLANE_MVT_v4f16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6005 if (RetVT.SimpleTy != MVT::v4f16)
6006 return 0;
6007 if ((Subtarget->hasNEON())) {
6008 return fastEmitInst_ri(ARM::VDUPLN32d, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6009 }
6010 return 0;
6011}
6012
6013unsigned fastEmit_ARMISD_VDUPLANE_MVT_v4bf16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6014 if (RetVT.SimpleTy != MVT::v4bf16)
6015 return 0;
6016 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
6017 return fastEmitInst_ri(ARM::VDUPLN16d, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6018 }
6019 return 0;
6020}
6021
6022unsigned fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v2f32_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6023 if ((Subtarget->hasNEON())) {
6024 return fastEmitInst_ri(ARM::VDUPLN32d, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6025 }
6026 return 0;
6027}
6028
6029unsigned fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v4f32_ri(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6030 if ((Subtarget->hasNEON())) {
6031 return fastEmitInst_ri(ARM::VDUPLN32q, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6032 }
6033 return 0;
6034}
6035
6036unsigned fastEmit_ARMISD_VDUPLANE_MVT_v2f32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6037switch (RetVT.SimpleTy) {
6038 case MVT::v2f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v2f32_ri(Op0, Op0IsKill, imm1);
6039 case MVT::v4f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_MVT_v4f32_ri(Op0, Op0IsKill, imm1);
6040 default: return 0;
6041}
6042}
6043
6044unsigned fastEmit_ARMISD_VDUPLANE_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6045 switch (VT.SimpleTy) {
6046 case MVT::v8i8: return fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri(RetVT, Op0, Op0IsKill, imm1);
6047 case MVT::v4i16: return fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri(RetVT, Op0, Op0IsKill, imm1);
6048 case MVT::v2i32: return fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri(RetVT, Op0, Op0IsKill, imm1);
6049 case MVT::v4f16: return fastEmit_ARMISD_VDUPLANE_MVT_v4f16_ri(RetVT, Op0, Op0IsKill, imm1);
6050 case MVT::v4bf16: return fastEmit_ARMISD_VDUPLANE_MVT_v4bf16_ri(RetVT, Op0, Op0IsKill, imm1);
6051 case MVT::v2f32: return fastEmit_ARMISD_VDUPLANE_MVT_v2f32_ri(RetVT, Op0, Op0IsKill, imm1);
6052 default: return 0;
6053 }
6054}
6055
6056// FastEmit functions for ARMISD::VGETLANEs.
6057
6058unsigned fastEmit_ARMISD_VGETLANEs_MVT_v8i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6059 if (RetVT.SimpleTy != MVT::i32)
6060 return 0;
6061 if ((Subtarget->hasNEON())) {
6062 return fastEmitInst_ri(ARM::VGETLNs8, &ARM::GPRRegClass, Op0, Op0IsKill, imm1);
6063 }
6064 return 0;
6065}
6066
6067unsigned fastEmit_ARMISD_VGETLANEs_MVT_v16i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6068 if (RetVT.SimpleTy != MVT::i32)
6069 return 0;
6070 if ((Subtarget->hasMVEIntegerOps())) {
6071 return fastEmitInst_ri(ARM::MVE_VMOV_from_lane_s8, &ARM::rGPRRegClass, Op0, Op0IsKill, imm1);
6072 }
6073 return 0;
6074}
6075
6076unsigned fastEmit_ARMISD_VGETLANEs_MVT_v4i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6077 if (RetVT.SimpleTy != MVT::i32)
6078 return 0;
6079 if ((Subtarget->hasNEON())) {
6080 return fastEmitInst_ri(ARM::VGETLNs16, &ARM::GPRRegClass, Op0, Op0IsKill, imm1);
6081 }
6082 return 0;
6083}
6084
6085unsigned fastEmit_ARMISD_VGETLANEs_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6086 if (RetVT.SimpleTy != MVT::i32)
6087 return 0;
6088 if ((Subtarget->hasMVEIntegerOps())) {
6089 return fastEmitInst_ri(ARM::MVE_VMOV_from_lane_s16, &ARM::rGPRRegClass, Op0, Op0IsKill, imm1);
6090 }
6091 return 0;
6092}
6093
6094unsigned fastEmit_ARMISD_VGETLANEs_MVT_v8f16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6095 if (RetVT.SimpleTy != MVT::i32)
6096 return 0;
6097 if ((Subtarget->hasMVEIntegerOps())) {
6098 return fastEmitInst_ri(ARM::MVE_VMOV_from_lane_s16, &ARM::rGPRRegClass, Op0, Op0IsKill, imm1);
6099 }
6100 return 0;
6101}
6102
6103unsigned fastEmit_ARMISD_VGETLANEs_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6104 switch (VT.SimpleTy) {
6105 case MVT::v8i8: return fastEmit_ARMISD_VGETLANEs_MVT_v8i8_ri(RetVT, Op0, Op0IsKill, imm1);
6106 case MVT::v16i8: return fastEmit_ARMISD_VGETLANEs_MVT_v16i8_ri(RetVT, Op0, Op0IsKill, imm1);
6107 case MVT::v4i16: return fastEmit_ARMISD_VGETLANEs_MVT_v4i16_ri(RetVT, Op0, Op0IsKill, imm1);
6108 case MVT::v8i16: return fastEmit_ARMISD_VGETLANEs_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
6109 case MVT::v8f16: return fastEmit_ARMISD_VGETLANEs_MVT_v8f16_ri(RetVT, Op0, Op0IsKill, imm1);
6110 default: return 0;
6111 }
6112}
6113
6114// FastEmit functions for ARMISD::VGETLANEu.
6115
6116unsigned fastEmit_ARMISD_VGETLANEu_MVT_v8i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6117 if (RetVT.SimpleTy != MVT::i32)
6118 return 0;
6119 if ((Subtarget->hasNEON())) {
6120 return fastEmitInst_ri(ARM::VGETLNu8, &ARM::GPRRegClass, Op0, Op0IsKill, imm1);
6121 }
6122 return 0;
6123}
6124
6125unsigned fastEmit_ARMISD_VGETLANEu_MVT_v16i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6126 if (RetVT.SimpleTy != MVT::i32)
6127 return 0;
6128 if ((Subtarget->hasMVEIntegerOps())) {
6129 return fastEmitInst_ri(ARM::MVE_VMOV_from_lane_u8, &ARM::rGPRRegClass, Op0, Op0IsKill, imm1);
6130 }
6131 return 0;
6132}
6133
6134unsigned fastEmit_ARMISD_VGETLANEu_MVT_v4i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6135 if (RetVT.SimpleTy != MVT::i32)
6136 return 0;
6137 if ((Subtarget->hasNEON())) {
6138 return fastEmitInst_ri(ARM::VGETLNu16, &ARM::GPRRegClass, Op0, Op0IsKill, imm1);
6139 }
6140 return 0;
6141}
6142
6143unsigned fastEmit_ARMISD_VGETLANEu_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6144 if (RetVT.SimpleTy != MVT::i32)
6145 return 0;
6146 if ((Subtarget->hasMVEIntegerOps())) {
6147 return fastEmitInst_ri(ARM::MVE_VMOV_from_lane_u16, &ARM::rGPRRegClass, Op0, Op0IsKill, imm1);
6148 }
6149 return 0;
6150}
6151
6152unsigned fastEmit_ARMISD_VGETLANEu_MVT_v8f16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6153 if (RetVT.SimpleTy != MVT::i32)
6154 return 0;
6155 if ((Subtarget->hasMVEIntegerOps())) {
6156 return fastEmitInst_ri(ARM::MVE_VMOV_from_lane_u16, &ARM::rGPRRegClass, Op0, Op0IsKill, imm1);
6157 }
6158 return 0;
6159}
6160
6161unsigned fastEmit_ARMISD_VGETLANEu_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6162 switch (VT.SimpleTy) {
6163 case MVT::v8i8: return fastEmit_ARMISD_VGETLANEu_MVT_v8i8_ri(RetVT, Op0, Op0IsKill, imm1);
6164 case MVT::v16i8: return fastEmit_ARMISD_VGETLANEu_MVT_v16i8_ri(RetVT, Op0, Op0IsKill, imm1);
6165 case MVT::v4i16: return fastEmit_ARMISD_VGETLANEu_MVT_v4i16_ri(RetVT, Op0, Op0IsKill, imm1);
6166 case MVT::v8i16: return fastEmit_ARMISD_VGETLANEu_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
6167 case MVT::v8f16: return fastEmit_ARMISD_VGETLANEu_MVT_v8f16_ri(RetVT, Op0, Op0IsKill, imm1);
6168 default: return 0;
6169 }
6170}
6171
6172// FastEmit functions for ARMISD::VQSHLsIMM.
6173
6174unsigned fastEmit_ARMISD_VQSHLsIMM_MVT_v8i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6175 if (RetVT.SimpleTy != MVT::v8i8)
6176 return 0;
6177 if ((Subtarget->hasNEON())) {
6178 return fastEmitInst_ri(ARM::VQSHLsiv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6179 }
6180 return 0;
6181}
6182
6183unsigned fastEmit_ARMISD_VQSHLsIMM_MVT_v16i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6184 if (RetVT.SimpleTy != MVT::v16i8)
6185 return 0;
6186 if ((Subtarget->hasNEON())) {
6187 return fastEmitInst_ri(ARM::VQSHLsiv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6188 }
6189 return 0;
6190}
6191
6192unsigned fastEmit_ARMISD_VQSHLsIMM_MVT_v4i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6193 if (RetVT.SimpleTy != MVT::v4i16)
6194 return 0;
6195 if ((Subtarget->hasNEON())) {
6196 return fastEmitInst_ri(ARM::VQSHLsiv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6197 }
6198 return 0;
6199}
6200
6201unsigned fastEmit_ARMISD_VQSHLsIMM_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6202 if (RetVT.SimpleTy != MVT::v8i16)
6203 return 0;
6204 if ((Subtarget->hasNEON())) {
6205 return fastEmitInst_ri(ARM::VQSHLsiv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6206 }
6207 return 0;
6208}
6209
6210unsigned fastEmit_ARMISD_VQSHLsIMM_MVT_v2i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6211 if (RetVT.SimpleTy != MVT::v2i32)
6212 return 0;
6213 if ((Subtarget->hasNEON())) {
6214 return fastEmitInst_ri(ARM::VQSHLsiv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6215 }
6216 return 0;
6217}
6218
6219unsigned fastEmit_ARMISD_VQSHLsIMM_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6220 if (RetVT.SimpleTy != MVT::v4i32)
6221 return 0;
6222 if ((Subtarget->hasNEON())) {
6223 return fastEmitInst_ri(ARM::VQSHLsiv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6224 }
6225 return 0;
6226}
6227
6228unsigned fastEmit_ARMISD_VQSHLsIMM_MVT_v1i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6229 if (RetVT.SimpleTy != MVT::v1i64)
6230 return 0;
6231 if ((Subtarget->hasNEON())) {
6232 return fastEmitInst_ri(ARM::VQSHLsiv1i64, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6233 }
6234 return 0;
6235}
6236
6237unsigned fastEmit_ARMISD_VQSHLsIMM_MVT_v2i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6238 if (RetVT.SimpleTy != MVT::v2i64)
6239 return 0;
6240 if ((Subtarget->hasNEON())) {
6241 return fastEmitInst_ri(ARM::VQSHLsiv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6242 }
6243 return 0;
6244}
6245
6246unsigned fastEmit_ARMISD_VQSHLsIMM_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6247 switch (VT.SimpleTy) {
6248 case MVT::v8i8: return fastEmit_ARMISD_VQSHLsIMM_MVT_v8i8_ri(RetVT, Op0, Op0IsKill, imm1);
6249 case MVT::v16i8: return fastEmit_ARMISD_VQSHLsIMM_MVT_v16i8_ri(RetVT, Op0, Op0IsKill, imm1);
6250 case MVT::v4i16: return fastEmit_ARMISD_VQSHLsIMM_MVT_v4i16_ri(RetVT, Op0, Op0IsKill, imm1);
6251 case MVT::v8i16: return fastEmit_ARMISD_VQSHLsIMM_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
6252 case MVT::v2i32: return fastEmit_ARMISD_VQSHLsIMM_MVT_v2i32_ri(RetVT, Op0, Op0IsKill, imm1);
6253 case MVT::v4i32: return fastEmit_ARMISD_VQSHLsIMM_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1);
6254 case MVT::v1i64: return fastEmit_ARMISD_VQSHLsIMM_MVT_v1i64_ri(RetVT, Op0, Op0IsKill, imm1);
6255 case MVT::v2i64: return fastEmit_ARMISD_VQSHLsIMM_MVT_v2i64_ri(RetVT, Op0, Op0IsKill, imm1);
6256 default: return 0;
6257 }
6258}
6259
6260// FastEmit functions for ARMISD::VQSHLsuIMM.
6261
6262unsigned fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6263 if (RetVT.SimpleTy != MVT::v8i8)
6264 return 0;
6265 if ((Subtarget->hasNEON())) {
6266 return fastEmitInst_ri(ARM::VQSHLsuv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6267 }
6268 return 0;
6269}
6270
6271unsigned fastEmit_ARMISD_VQSHLsuIMM_MVT_v16i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6272 if (RetVT.SimpleTy != MVT::v16i8)
6273 return 0;
6274 if ((Subtarget->hasNEON())) {
6275 return fastEmitInst_ri(ARM::VQSHLsuv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6276 }
6277 return 0;
6278}
6279
6280unsigned fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6281 if (RetVT.SimpleTy != MVT::v4i16)
6282 return 0;
6283 if ((Subtarget->hasNEON())) {
6284 return fastEmitInst_ri(ARM::VQSHLsuv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6285 }
6286 return 0;
6287}
6288
6289unsigned fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6290 if (RetVT.SimpleTy != MVT::v8i16)
6291 return 0;
6292 if ((Subtarget->hasNEON())) {
6293 return fastEmitInst_ri(ARM::VQSHLsuv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6294 }
6295 return 0;
6296}
6297
6298unsigned fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6299 if (RetVT.SimpleTy != MVT::v2i32)
6300 return 0;
6301 if ((Subtarget->hasNEON())) {
6302 return fastEmitInst_ri(ARM::VQSHLsuv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6303 }
6304 return 0;
6305}
6306
6307unsigned fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6308 if (RetVT.SimpleTy != MVT::v4i32)
6309 return 0;
6310 if ((Subtarget->hasNEON())) {
6311 return fastEmitInst_ri(ARM::VQSHLsuv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6312 }
6313 return 0;
6314}
6315
6316unsigned fastEmit_ARMISD_VQSHLsuIMM_MVT_v1i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6317 if (RetVT.SimpleTy != MVT::v1i64)
6318 return 0;
6319 if ((Subtarget->hasNEON())) {
6320 return fastEmitInst_ri(ARM::VQSHLsuv1i64, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6321 }
6322 return 0;
6323}
6324
6325unsigned fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6326 if (RetVT.SimpleTy != MVT::v2i64)
6327 return 0;
6328 if ((Subtarget->hasNEON())) {
6329 return fastEmitInst_ri(ARM::VQSHLsuv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6330 }
6331 return 0;
6332}
6333
6334unsigned fastEmit_ARMISD_VQSHLsuIMM_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6335 switch (VT.SimpleTy) {
6336 case MVT::v8i8: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i8_ri(RetVT, Op0, Op0IsKill, imm1);
6337 case MVT::v16i8: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v16i8_ri(RetVT, Op0, Op0IsKill, imm1);
6338 case MVT::v4i16: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i16_ri(RetVT, Op0, Op0IsKill, imm1);
6339 case MVT::v8i16: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
6340 case MVT::v2i32: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i32_ri(RetVT, Op0, Op0IsKill, imm1);
6341 case MVT::v4i32: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1);
6342 case MVT::v1i64: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v1i64_ri(RetVT, Op0, Op0IsKill, imm1);
6343 case MVT::v2i64: return fastEmit_ARMISD_VQSHLsuIMM_MVT_v2i64_ri(RetVT, Op0, Op0IsKill, imm1);
6344 default: return 0;
6345 }
6346}
6347
6348// FastEmit functions for ARMISD::VQSHLuIMM.
6349
6350unsigned fastEmit_ARMISD_VQSHLuIMM_MVT_v8i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6351 if (RetVT.SimpleTy != MVT::v8i8)
6352 return 0;
6353 if ((Subtarget->hasNEON())) {
6354 return fastEmitInst_ri(ARM::VQSHLuiv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6355 }
6356 return 0;
6357}
6358
6359unsigned fastEmit_ARMISD_VQSHLuIMM_MVT_v16i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6360 if (RetVT.SimpleTy != MVT::v16i8)
6361 return 0;
6362 if ((Subtarget->hasNEON())) {
6363 return fastEmitInst_ri(ARM::VQSHLuiv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6364 }
6365 return 0;
6366}
6367
6368unsigned fastEmit_ARMISD_VQSHLuIMM_MVT_v4i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6369 if (RetVT.SimpleTy != MVT::v4i16)
6370 return 0;
6371 if ((Subtarget->hasNEON())) {
6372 return fastEmitInst_ri(ARM::VQSHLuiv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6373 }
6374 return 0;
6375}
6376
6377unsigned fastEmit_ARMISD_VQSHLuIMM_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6378 if (RetVT.SimpleTy != MVT::v8i16)
6379 return 0;
6380 if ((Subtarget->hasNEON())) {
6381 return fastEmitInst_ri(ARM::VQSHLuiv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6382 }
6383 return 0;
6384}
6385
6386unsigned fastEmit_ARMISD_VQSHLuIMM_MVT_v2i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6387 if (RetVT.SimpleTy != MVT::v2i32)
6388 return 0;
6389 if ((Subtarget->hasNEON())) {
6390 return fastEmitInst_ri(ARM::VQSHLuiv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6391 }
6392 return 0;
6393}
6394
6395unsigned fastEmit_ARMISD_VQSHLuIMM_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6396 if (RetVT.SimpleTy != MVT::v4i32)
6397 return 0;
6398 if ((Subtarget->hasNEON())) {
6399 return fastEmitInst_ri(ARM::VQSHLuiv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6400 }
6401 return 0;
6402}
6403
6404unsigned fastEmit_ARMISD_VQSHLuIMM_MVT_v1i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6405 if (RetVT.SimpleTy != MVT::v1i64)
6406 return 0;
6407 if ((Subtarget->hasNEON())) {
6408 return fastEmitInst_ri(ARM::VQSHLuiv1i64, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6409 }
6410 return 0;
6411}
6412
6413unsigned fastEmit_ARMISD_VQSHLuIMM_MVT_v2i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6414 if (RetVT.SimpleTy != MVT::v2i64)
6415 return 0;
6416 if ((Subtarget->hasNEON())) {
6417 return fastEmitInst_ri(ARM::VQSHLuiv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6418 }
6419 return 0;
6420}
6421
6422unsigned fastEmit_ARMISD_VQSHLuIMM_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6423 switch (VT.SimpleTy) {
6424 case MVT::v8i8: return fastEmit_ARMISD_VQSHLuIMM_MVT_v8i8_ri(RetVT, Op0, Op0IsKill, imm1);
6425 case MVT::v16i8: return fastEmit_ARMISD_VQSHLuIMM_MVT_v16i8_ri(RetVT, Op0, Op0IsKill, imm1);
6426 case MVT::v4i16: return fastEmit_ARMISD_VQSHLuIMM_MVT_v4i16_ri(RetVT, Op0, Op0IsKill, imm1);
6427 case MVT::v8i16: return fastEmit_ARMISD_VQSHLuIMM_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
6428 case MVT::v2i32: return fastEmit_ARMISD_VQSHLuIMM_MVT_v2i32_ri(RetVT, Op0, Op0IsKill, imm1);
6429 case MVT::v4i32: return fastEmit_ARMISD_VQSHLuIMM_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1);
6430 case MVT::v1i64: return fastEmit_ARMISD_VQSHLuIMM_MVT_v1i64_ri(RetVT, Op0, Op0IsKill, imm1);
6431 case MVT::v2i64: return fastEmit_ARMISD_VQSHLuIMM_MVT_v2i64_ri(RetVT, Op0, Op0IsKill, imm1);
6432 default: return 0;
6433 }
6434}
6435
6436// FastEmit functions for ARMISD::VRSHRsIMM.
6437
6438unsigned fastEmit_ARMISD_VRSHRsIMM_MVT_v8i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6439 if (RetVT.SimpleTy != MVT::v8i8)
6440 return 0;
6441 if ((Subtarget->hasNEON())) {
6442 return fastEmitInst_ri(ARM::VRSHRsv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6443 }
6444 return 0;
6445}
6446
6447unsigned fastEmit_ARMISD_VRSHRsIMM_MVT_v16i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6448 if (RetVT.SimpleTy != MVT::v16i8)
6449 return 0;
6450 if ((Subtarget->hasNEON())) {
6451 return fastEmitInst_ri(ARM::VRSHRsv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6452 }
6453 return 0;
6454}
6455
6456unsigned fastEmit_ARMISD_VRSHRsIMM_MVT_v4i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6457 if (RetVT.SimpleTy != MVT::v4i16)
6458 return 0;
6459 if ((Subtarget->hasNEON())) {
6460 return fastEmitInst_ri(ARM::VRSHRsv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6461 }
6462 return 0;
6463}
6464
6465unsigned fastEmit_ARMISD_VRSHRsIMM_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6466 if (RetVT.SimpleTy != MVT::v8i16)
6467 return 0;
6468 if ((Subtarget->hasNEON())) {
6469 return fastEmitInst_ri(ARM::VRSHRsv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6470 }
6471 return 0;
6472}
6473
6474unsigned fastEmit_ARMISD_VRSHRsIMM_MVT_v2i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6475 if (RetVT.SimpleTy != MVT::v2i32)
6476 return 0;
6477 if ((Subtarget->hasNEON())) {
6478 return fastEmitInst_ri(ARM::VRSHRsv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6479 }
6480 return 0;
6481}
6482
6483unsigned fastEmit_ARMISD_VRSHRsIMM_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6484 if (RetVT.SimpleTy != MVT::v4i32)
6485 return 0;
6486 if ((Subtarget->hasNEON())) {
6487 return fastEmitInst_ri(ARM::VRSHRsv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6488 }
6489 return 0;
6490}
6491
6492unsigned fastEmit_ARMISD_VRSHRsIMM_MVT_v1i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6493 if (RetVT.SimpleTy != MVT::v1i64)
6494 return 0;
6495 if ((Subtarget->hasNEON())) {
6496 return fastEmitInst_ri(ARM::VRSHRsv1i64, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6497 }
6498 return 0;
6499}
6500
6501unsigned fastEmit_ARMISD_VRSHRsIMM_MVT_v2i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6502 if (RetVT.SimpleTy != MVT::v2i64)
6503 return 0;
6504 if ((Subtarget->hasNEON())) {
6505 return fastEmitInst_ri(ARM::VRSHRsv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6506 }
6507 return 0;
6508}
6509
6510unsigned fastEmit_ARMISD_VRSHRsIMM_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6511 switch (VT.SimpleTy) {
6512 case MVT::v8i8: return fastEmit_ARMISD_VRSHRsIMM_MVT_v8i8_ri(RetVT, Op0, Op0IsKill, imm1);
6513 case MVT::v16i8: return fastEmit_ARMISD_VRSHRsIMM_MVT_v16i8_ri(RetVT, Op0, Op0IsKill, imm1);
6514 case MVT::v4i16: return fastEmit_ARMISD_VRSHRsIMM_MVT_v4i16_ri(RetVT, Op0, Op0IsKill, imm1);
6515 case MVT::v8i16: return fastEmit_ARMISD_VRSHRsIMM_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
6516 case MVT::v2i32: return fastEmit_ARMISD_VRSHRsIMM_MVT_v2i32_ri(RetVT, Op0, Op0IsKill, imm1);
6517 case MVT::v4i32: return fastEmit_ARMISD_VRSHRsIMM_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1);
6518 case MVT::v1i64: return fastEmit_ARMISD_VRSHRsIMM_MVT_v1i64_ri(RetVT, Op0, Op0IsKill, imm1);
6519 case MVT::v2i64: return fastEmit_ARMISD_VRSHRsIMM_MVT_v2i64_ri(RetVT, Op0, Op0IsKill, imm1);
6520 default: return 0;
6521 }
6522}
6523
6524// FastEmit functions for ARMISD::VRSHRuIMM.
6525
6526unsigned fastEmit_ARMISD_VRSHRuIMM_MVT_v8i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6527 if (RetVT.SimpleTy != MVT::v8i8)
6528 return 0;
6529 if ((Subtarget->hasNEON())) {
6530 return fastEmitInst_ri(ARM::VRSHRuv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6531 }
6532 return 0;
6533}
6534
6535unsigned fastEmit_ARMISD_VRSHRuIMM_MVT_v16i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6536 if (RetVT.SimpleTy != MVT::v16i8)
6537 return 0;
6538 if ((Subtarget->hasNEON())) {
6539 return fastEmitInst_ri(ARM::VRSHRuv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6540 }
6541 return 0;
6542}
6543
6544unsigned fastEmit_ARMISD_VRSHRuIMM_MVT_v4i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6545 if (RetVT.SimpleTy != MVT::v4i16)
6546 return 0;
6547 if ((Subtarget->hasNEON())) {
6548 return fastEmitInst_ri(ARM::VRSHRuv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6549 }
6550 return 0;
6551}
6552
6553unsigned fastEmit_ARMISD_VRSHRuIMM_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6554 if (RetVT.SimpleTy != MVT::v8i16)
6555 return 0;
6556 if ((Subtarget->hasNEON())) {
6557 return fastEmitInst_ri(ARM::VRSHRuv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6558 }
6559 return 0;
6560}
6561
6562unsigned fastEmit_ARMISD_VRSHRuIMM_MVT_v2i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6563 if (RetVT.SimpleTy != MVT::v2i32)
6564 return 0;
6565 if ((Subtarget->hasNEON())) {
6566 return fastEmitInst_ri(ARM::VRSHRuv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6567 }
6568 return 0;
6569}
6570
6571unsigned fastEmit_ARMISD_VRSHRuIMM_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6572 if (RetVT.SimpleTy != MVT::v4i32)
6573 return 0;
6574 if ((Subtarget->hasNEON())) {
6575 return fastEmitInst_ri(ARM::VRSHRuv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6576 }
6577 return 0;
6578}
6579
6580unsigned fastEmit_ARMISD_VRSHRuIMM_MVT_v1i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6581 if (RetVT.SimpleTy != MVT::v1i64)
6582 return 0;
6583 if ((Subtarget->hasNEON())) {
6584 return fastEmitInst_ri(ARM::VRSHRuv1i64, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6585 }
6586 return 0;
6587}
6588
6589unsigned fastEmit_ARMISD_VRSHRuIMM_MVT_v2i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6590 if (RetVT.SimpleTy != MVT::v2i64)
6591 return 0;
6592 if ((Subtarget->hasNEON())) {
6593 return fastEmitInst_ri(ARM::VRSHRuv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6594 }
6595 return 0;
6596}
6597
6598unsigned fastEmit_ARMISD_VRSHRuIMM_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6599 switch (VT.SimpleTy) {
6600 case MVT::v8i8: return fastEmit_ARMISD_VRSHRuIMM_MVT_v8i8_ri(RetVT, Op0, Op0IsKill, imm1);
6601 case MVT::v16i8: return fastEmit_ARMISD_VRSHRuIMM_MVT_v16i8_ri(RetVT, Op0, Op0IsKill, imm1);
6602 case MVT::v4i16: return fastEmit_ARMISD_VRSHRuIMM_MVT_v4i16_ri(RetVT, Op0, Op0IsKill, imm1);
6603 case MVT::v8i16: return fastEmit_ARMISD_VRSHRuIMM_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
6604 case MVT::v2i32: return fastEmit_ARMISD_VRSHRuIMM_MVT_v2i32_ri(RetVT, Op0, Op0IsKill, imm1);
6605 case MVT::v4i32: return fastEmit_ARMISD_VRSHRuIMM_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1);
6606 case MVT::v1i64: return fastEmit_ARMISD_VRSHRuIMM_MVT_v1i64_ri(RetVT, Op0, Op0IsKill, imm1);
6607 case MVT::v2i64: return fastEmit_ARMISD_VRSHRuIMM_MVT_v2i64_ri(RetVT, Op0, Op0IsKill, imm1);
6608 default: return 0;
6609 }
6610}
6611
6612// FastEmit functions for ARMISD::VSHLIMM.
6613
6614unsigned fastEmit_ARMISD_VSHLIMM_MVT_v8i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6615 if (RetVT.SimpleTy != MVT::v8i8)
6616 return 0;
6617 if ((Subtarget->hasNEON())) {
6618 return fastEmitInst_ri(ARM::VSHLiv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6619 }
6620 return 0;
6621}
6622
6623unsigned fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6624 if (RetVT.SimpleTy != MVT::v16i8)
6625 return 0;
6626 if ((Subtarget->hasNEON())) {
6627 return fastEmitInst_ri(ARM::VSHLiv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6628 }
6629 return 0;
6630}
6631
6632unsigned fastEmit_ARMISD_VSHLIMM_MVT_v4i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6633 if (RetVT.SimpleTy != MVT::v4i16)
6634 return 0;
6635 if ((Subtarget->hasNEON())) {
6636 return fastEmitInst_ri(ARM::VSHLiv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6637 }
6638 return 0;
6639}
6640
6641unsigned fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6642 if (RetVT.SimpleTy != MVT::v8i16)
6643 return 0;
6644 if ((Subtarget->hasNEON())) {
6645 return fastEmitInst_ri(ARM::VSHLiv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6646 }
6647 return 0;
6648}
6649
6650unsigned fastEmit_ARMISD_VSHLIMM_MVT_v2i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6651 if (RetVT.SimpleTy != MVT::v2i32)
6652 return 0;
6653 if ((Subtarget->hasNEON())) {
6654 return fastEmitInst_ri(ARM::VSHLiv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6655 }
6656 return 0;
6657}
6658
6659unsigned fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6660 if (RetVT.SimpleTy != MVT::v4i32)
6661 return 0;
6662 if ((Subtarget->hasNEON())) {
6663 return fastEmitInst_ri(ARM::VSHLiv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6664 }
6665 return 0;
6666}
6667
6668unsigned fastEmit_ARMISD_VSHLIMM_MVT_v1i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6669 if (RetVT.SimpleTy != MVT::v1i64)
6670 return 0;
6671 if ((Subtarget->hasNEON())) {
6672 return fastEmitInst_ri(ARM::VSHLiv1i64, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6673 }
6674 return 0;
6675}
6676
6677unsigned fastEmit_ARMISD_VSHLIMM_MVT_v2i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6678 if (RetVT.SimpleTy != MVT::v2i64)
6679 return 0;
6680 if ((Subtarget->hasNEON())) {
6681 return fastEmitInst_ri(ARM::VSHLiv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6682 }
6683 return 0;
6684}
6685
6686unsigned fastEmit_ARMISD_VSHLIMM_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6687 switch (VT.SimpleTy) {
6688 case MVT::v8i8: return fastEmit_ARMISD_VSHLIMM_MVT_v8i8_ri(RetVT, Op0, Op0IsKill, imm1);
6689 case MVT::v16i8: return fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri(RetVT, Op0, Op0IsKill, imm1);
6690 case MVT::v4i16: return fastEmit_ARMISD_VSHLIMM_MVT_v4i16_ri(RetVT, Op0, Op0IsKill, imm1);
6691 case MVT::v8i16: return fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
6692 case MVT::v2i32: return fastEmit_ARMISD_VSHLIMM_MVT_v2i32_ri(RetVT, Op0, Op0IsKill, imm1);
6693 case MVT::v4i32: return fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1);
6694 case MVT::v1i64: return fastEmit_ARMISD_VSHLIMM_MVT_v1i64_ri(RetVT, Op0, Op0IsKill, imm1);
6695 case MVT::v2i64: return fastEmit_ARMISD_VSHLIMM_MVT_v2i64_ri(RetVT, Op0, Op0IsKill, imm1);
6696 default: return 0;
6697 }
6698}
6699
6700// FastEmit functions for ARMISD::VSHRsIMM.
6701
6702unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v8i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6703 if (RetVT.SimpleTy != MVT::v8i8)
6704 return 0;
6705 if ((Subtarget->hasNEON())) {
6706 return fastEmitInst_ri(ARM::VSHRsv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6707 }
6708 return 0;
6709}
6710
6711unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6712 if (RetVT.SimpleTy != MVT::v16i8)
6713 return 0;
6714 if ((Subtarget->hasNEON())) {
6715 return fastEmitInst_ri(ARM::VSHRsv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6716 }
6717 return 0;
6718}
6719
6720unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v4i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6721 if (RetVT.SimpleTy != MVT::v4i16)
6722 return 0;
6723 if ((Subtarget->hasNEON())) {
6724 return fastEmitInst_ri(ARM::VSHRsv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6725 }
6726 return 0;
6727}
6728
6729unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6730 if (RetVT.SimpleTy != MVT::v8i16)
6731 return 0;
6732 if ((Subtarget->hasNEON())) {
6733 return fastEmitInst_ri(ARM::VSHRsv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6734 }
6735 return 0;
6736}
6737
6738unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v2i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6739 if (RetVT.SimpleTy != MVT::v2i32)
6740 return 0;
6741 if ((Subtarget->hasNEON())) {
6742 return fastEmitInst_ri(ARM::VSHRsv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6743 }
6744 return 0;
6745}
6746
6747unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6748 if (RetVT.SimpleTy != MVT::v4i32)
6749 return 0;
6750 if ((Subtarget->hasNEON())) {
6751 return fastEmitInst_ri(ARM::VSHRsv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6752 }
6753 return 0;
6754}
6755
6756unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v1i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6757 if (RetVT.SimpleTy != MVT::v1i64)
6758 return 0;
6759 if ((Subtarget->hasNEON())) {
6760 return fastEmitInst_ri(ARM::VSHRsv1i64, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6761 }
6762 return 0;
6763}
6764
6765unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v2i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6766 if (RetVT.SimpleTy != MVT::v2i64)
6767 return 0;
6768 if ((Subtarget->hasNEON())) {
6769 return fastEmitInst_ri(ARM::VSHRsv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6770 }
6771 return 0;
6772}
6773
6774unsigned fastEmit_ARMISD_VSHRsIMM_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6775 switch (VT.SimpleTy) {
6776 case MVT::v8i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i8_ri(RetVT, Op0, Op0IsKill, imm1);
6777 case MVT::v16i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri(RetVT, Op0, Op0IsKill, imm1);
6778 case MVT::v4i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i16_ri(RetVT, Op0, Op0IsKill, imm1);
6779 case MVT::v8i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
6780 case MVT::v2i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v2i32_ri(RetVT, Op0, Op0IsKill, imm1);
6781 case MVT::v4i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1);
6782 case MVT::v1i64: return fastEmit_ARMISD_VSHRsIMM_MVT_v1i64_ri(RetVT, Op0, Op0IsKill, imm1);
6783 case MVT::v2i64: return fastEmit_ARMISD_VSHRsIMM_MVT_v2i64_ri(RetVT, Op0, Op0IsKill, imm1);
6784 default: return 0;
6785 }
6786}
6787
6788// FastEmit functions for ARMISD::VSHRuIMM.
6789
6790unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v8i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6791 if (RetVT.SimpleTy != MVT::v8i8)
6792 return 0;
6793 if ((Subtarget->hasNEON())) {
6794 return fastEmitInst_ri(ARM::VSHRuv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6795 }
6796 return 0;
6797}
6798
6799unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6800 if (RetVT.SimpleTy != MVT::v16i8)
6801 return 0;
6802 if ((Subtarget->hasNEON())) {
6803 return fastEmitInst_ri(ARM::VSHRuv16i8, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6804 }
6805 return 0;
6806}
6807
6808unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v4i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6809 if (RetVT.SimpleTy != MVT::v4i16)
6810 return 0;
6811 if ((Subtarget->hasNEON())) {
6812 return fastEmitInst_ri(ARM::VSHRuv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6813 }
6814 return 0;
6815}
6816
6817unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6818 if (RetVT.SimpleTy != MVT::v8i16)
6819 return 0;
6820 if ((Subtarget->hasNEON())) {
6821 return fastEmitInst_ri(ARM::VSHRuv8i16, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6822 }
6823 return 0;
6824}
6825
6826unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v2i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6827 if (RetVT.SimpleTy != MVT::v2i32)
6828 return 0;
6829 if ((Subtarget->hasNEON())) {
6830 return fastEmitInst_ri(ARM::VSHRuv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6831 }
6832 return 0;
6833}
6834
6835unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6836 if (RetVT.SimpleTy != MVT::v4i32)
6837 return 0;
6838 if ((Subtarget->hasNEON())) {
6839 return fastEmitInst_ri(ARM::VSHRuv4i32, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6840 }
6841 return 0;
6842}
6843
6844unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v1i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6845 if (RetVT.SimpleTy != MVT::v1i64)
6846 return 0;
6847 if ((Subtarget->hasNEON())) {
6848 return fastEmitInst_ri(ARM::VSHRuv1i64, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
6849 }
6850 return 0;
6851}
6852
6853unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v2i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6854 if (RetVT.SimpleTy != MVT::v2i64)
6855 return 0;
6856 if ((Subtarget->hasNEON())) {
6857 return fastEmitInst_ri(ARM::VSHRuv2i64, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
6858 }
6859 return 0;
6860}
6861
6862unsigned fastEmit_ARMISD_VSHRuIMM_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6863 switch (VT.SimpleTy) {
6864 case MVT::v8i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i8_ri(RetVT, Op0, Op0IsKill, imm1);
6865 case MVT::v16i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri(RetVT, Op0, Op0IsKill, imm1);
6866 case MVT::v4i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i16_ri(RetVT, Op0, Op0IsKill, imm1);
6867 case MVT::v8i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1);
6868 case MVT::v2i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v2i32_ri(RetVT, Op0, Op0IsKill, imm1);
6869 case MVT::v4i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1);
6870 case MVT::v1i64: return fastEmit_ARMISD_VSHRuIMM_MVT_v1i64_ri(RetVT, Op0, Op0IsKill, imm1);
6871 case MVT::v2i64: return fastEmit_ARMISD_VSHRuIMM_MVT_v2i64_ri(RetVT, Op0, Op0IsKill, imm1);
6872 default: return 0;
6873 }
6874}
6875
6876// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
6877
6878unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6879 if (RetVT.SimpleTy != MVT::i32)
6880 return 0;
6881 if ((Subtarget->hasFPRegs()) && (!Subtarget->hasSlowVGETLNi32())) {
6882 return fastEmitInst_ri(ARM::VGETLNi32, &ARM::GPRRegClass, Op0, Op0IsKill, imm1);
6883 }
6884 return 0;
6885}
6886
6887unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6888 switch (VT.SimpleTy) {
6889 case MVT::v2i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i32_ri(RetVT, Op0, Op0IsKill, imm1);
6890 default: return 0;
6891 }
6892}
6893
6894// FastEmit functions for ISD::SHL.
6895
6896unsigned fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6897 if (RetVT.SimpleTy != MVT::i32)
6898 return 0;
6899 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
6900 return fastEmitInst_ri(ARM::tLSLri, &ARM::tGPRRegClass, Op0, Op0IsKill, imm1);
6901 }
6902 return 0;
6903}
6904
6905unsigned fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6906 switch (VT.SimpleTy) {
6907 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
6908 default: return 0;
6909 }
6910}
6911
6912// Top-level FastEmit function.
6913
6914unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) override {
6915 if (VT == MVT::i32 && Predicate_mod_imm(imm1))
6916 if (unsigned Reg = fastEmit_ri_Predicate_mod_imm(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
6917 return Reg;
6918
6919 if (VT == MVT::i32 && Predicate_imm0_7(imm1))
6920 if (unsigned Reg = fastEmit_ri_Predicate_imm0_7(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
6921 return Reg;
6922
6923 if (VT == MVT::i32 && Predicate_imm8_255(imm1))
6924 if (unsigned Reg = fastEmit_ri_Predicate_imm8_255(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
6925 return Reg;
6926
6927 if (VT == MVT::i32 && Predicate_imm0_255(imm1))
6928 if (unsigned Reg = fastEmit_ri_Predicate_imm0_255(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
6929 return Reg;
6930
6931 if (VT == MVT::i32 && Predicate_t2_so_imm(imm1))
6932 if (unsigned Reg = fastEmit_ri_Predicate_t2_so_imm(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
6933 return Reg;
6934
6935 if (VT == MVT::i32 && Predicate_imm0_4095(imm1))
6936 if (unsigned Reg = fastEmit_ri_Predicate_imm0_4095(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
6937 return Reg;
6938
6939 if (VT == MVT::i32 && Predicate_imm1_31(imm1))
6940 if (unsigned Reg = fastEmit_ri_Predicate_imm1_31(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
6941 return Reg;
6942
6943 if (VT == MVT::i32 && Predicate_imm0_31(imm1))
6944 if (unsigned Reg = fastEmit_ri_Predicate_imm0_31(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
6945 return Reg;
6946
6947 if (VT == MVT::i32 && Predicate_shr_imm8(imm1))
6948 if (unsigned Reg = fastEmit_ri_Predicate_shr_imm8(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
6949 return Reg;
6950
6951 if (VT == MVT::i32 && Predicate_shr_imm16(imm1))
6952 if (unsigned Reg = fastEmit_ri_Predicate_shr_imm16(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
6953 return Reg;
6954
6955 if (VT == MVT::i32 && Predicate_shr_imm32(imm1))
6956 if (unsigned Reg = fastEmit_ri_Predicate_shr_imm32(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
6957 return Reg;
6958
6959 if (VT == MVT::i32 && Predicate_VectorIndex32(imm1))
6960 if (unsigned Reg = fastEmit_ri_Predicate_VectorIndex32(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
6961 return Reg;
6962
6963 if (VT == MVT::i32 && Predicate_imm0_15(imm1))
6964 if (unsigned Reg = fastEmit_ri_Predicate_imm0_15(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
6965 return Reg;
6966
6967 switch (Opcode) {
6968 case ARMISD::PIC_ADD: return fastEmit_ARMISD_PIC_ADD_ri(VT, RetVT, Op0, Op0IsKill, imm1);
6969 case ARMISD::VDUPLANE: return fastEmit_ARMISD_VDUPLANE_ri(VT, RetVT, Op0, Op0IsKill, imm1);
6970 case ARMISD::VGETLANEs: return fastEmit_ARMISD_VGETLANEs_ri(VT, RetVT, Op0, Op0IsKill, imm1);
6971 case ARMISD::VGETLANEu: return fastEmit_ARMISD_VGETLANEu_ri(VT, RetVT, Op0, Op0IsKill, imm1);
6972 case ARMISD::VQSHLsIMM: return fastEmit_ARMISD_VQSHLsIMM_ri(VT, RetVT, Op0, Op0IsKill, imm1);
6973 case ARMISD::VQSHLsuIMM: return fastEmit_ARMISD_VQSHLsuIMM_ri(VT, RetVT, Op0, Op0IsKill, imm1);
6974 case ARMISD::VQSHLuIMM: return fastEmit_ARMISD_VQSHLuIMM_ri(VT, RetVT, Op0, Op0IsKill, imm1);
6975 case ARMISD::VRSHRsIMM: return fastEmit_ARMISD_VRSHRsIMM_ri(VT, RetVT, Op0, Op0IsKill, imm1);
6976 case ARMISD::VRSHRuIMM: return fastEmit_ARMISD_VRSHRuIMM_ri(VT, RetVT, Op0, Op0IsKill, imm1);
6977 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri(VT, RetVT, Op0, Op0IsKill, imm1);
6978 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri(VT, RetVT, Op0, Op0IsKill, imm1);
6979 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri(VT, RetVT, Op0, Op0IsKill, imm1);
6980 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, Op0IsKill, imm1);
6981 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, Op0IsKill, imm1);
6982 default: return 0;
6983 }
6984}
6985
6986// FastEmit functions for ARMISD::CMN.
6987
6988unsigned fastEmit_ARMISD_CMN_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6989 if (RetVT.SimpleTy != MVT::isVoid)
6990 return 0;
6991 if ((!Subtarget->isThumb())) {
6992 return fastEmitInst_ri(ARM::CMNri, &ARM::GPRRegClass, Op0, Op0IsKill, imm1);
6993 }
6994 return 0;
6995}
6996
6997unsigned fastEmit_ARMISD_CMN_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
6998 switch (VT.SimpleTy) {
6999 case MVT::i32: return fastEmit_ARMISD_CMN_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, Op0IsKill, imm1);
7000 default: return 0;
7001 }
7002}
7003
7004// FastEmit functions for ARMISD::CMP.
7005
7006unsigned fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7007 if (RetVT.SimpleTy != MVT::isVoid)
7008 return 0;
7009 if ((!Subtarget->isThumb())) {
7010 return fastEmitInst_ri(ARM::CMPri, &ARM::GPRRegClass, Op0, Op0IsKill, imm1);
7011 }
7012 return 0;
7013}
7014
7015unsigned fastEmit_ARMISD_CMP_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7016 switch (VT.SimpleTy) {
7017 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, Op0IsKill, imm1);
7018 default: return 0;
7019 }
7020}
7021
7022// FastEmit functions for ARMISD::CMPZ.
7023
7024unsigned fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7025 if (RetVT.SimpleTy != MVT::isVoid)
7026 return 0;
7027 if ((!Subtarget->isThumb())) {
7028 return fastEmitInst_ri(ARM::CMPri, &ARM::GPRRegClass, Op0, Op0IsKill, imm1);
7029 }
7030 return 0;
7031}
7032
7033unsigned fastEmit_ARMISD_CMPZ_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7034 switch (VT.SimpleTy) {
7035 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, Op0IsKill, imm1);
7036 default: return 0;
7037 }
7038}
7039
7040// FastEmit functions for ARMISD::SUBS.
7041
7042unsigned fastEmit_ARMISD_SUBS_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7043 if (RetVT.SimpleTy != MVT::i32)
7044 return 0;
7045 if ((!Subtarget->isThumb())) {
7046 return fastEmitInst_ri(ARM::SUBSri, &ARM::GPRRegClass, Op0, Op0IsKill, imm1);
7047 }
7048 return 0;
7049}
7050
7051unsigned fastEmit_ARMISD_SUBS_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7052 switch (VT.SimpleTy) {
7053 case MVT::i32: return fastEmit_ARMISD_SUBS_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, Op0IsKill, imm1);
7054 default: return 0;
7055 }
7056}
7057
7058// FastEmit functions for ISD::ADD.
7059
7060unsigned fastEmit_ISD_ADD_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7061 if (RetVT.SimpleTy != MVT::i32)
7062 return 0;
7063 if ((!Subtarget->isThumb())) {
7064 return fastEmitInst_ri(ARM::ADDri, &ARM::GPRRegClass, Op0, Op0IsKill, imm1);
7065 }
7066 return 0;
7067}
7068
7069unsigned fastEmit_ISD_ADD_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7070 switch (VT.SimpleTy) {
7071 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, Op0IsKill, imm1);
7072 default: return 0;
7073 }
7074}
7075
7076// FastEmit functions for ISD::AND.
7077
7078unsigned fastEmit_ISD_AND_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7079 if (RetVT.SimpleTy != MVT::i32)
7080 return 0;
7081 if ((!Subtarget->isThumb())) {
7082 return fastEmitInst_ri(ARM::ANDri, &ARM::GPRRegClass, Op0, Op0IsKill, imm1);
7083 }
7084 return 0;
7085}
7086
7087unsigned fastEmit_ISD_AND_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7088 switch (VT.SimpleTy) {
7089 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, Op0IsKill, imm1);
7090 default: return 0;
7091 }
7092}
7093
7094// FastEmit functions for ISD::OR.
7095
7096unsigned fastEmit_ISD_OR_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7097 if (RetVT.SimpleTy != MVT::i32)
7098 return 0;
7099 if ((!Subtarget->isThumb())) {
7100 return fastEmitInst_ri(ARM::ORRri, &ARM::GPRRegClass, Op0, Op0IsKill, imm1);
7101 }
7102 return 0;
7103}
7104
7105unsigned fastEmit_ISD_OR_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7106 switch (VT.SimpleTy) {
7107 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, Op0IsKill, imm1);
7108 default: return 0;
7109 }
7110}
7111
7112// FastEmit functions for ISD::SUB.
7113
7114unsigned fastEmit_ISD_SUB_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7115 if (RetVT.SimpleTy != MVT::i32)
7116 return 0;
7117 if ((!Subtarget->isThumb())) {
7118 return fastEmitInst_ri(ARM::SUBri, &ARM::GPRRegClass, Op0, Op0IsKill, imm1);
7119 }
7120 return 0;
7121}
7122
7123unsigned fastEmit_ISD_SUB_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7124 switch (VT.SimpleTy) {
7125 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, Op0IsKill, imm1);
7126 default: return 0;
7127 }
7128}
7129
7130// FastEmit functions for ISD::XOR.
7131
7132unsigned fastEmit_ISD_XOR_MVT_i32_ri_Predicate_mod_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7133 if (RetVT.SimpleTy != MVT::i32)
7134 return 0;
7135 if ((!Subtarget->isThumb())) {
7136 return fastEmitInst_ri(ARM::EORri, &ARM::GPRRegClass, Op0, Op0IsKill, imm1);
7137 }
7138 return 0;
7139}
7140
7141unsigned fastEmit_ISD_XOR_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7142 switch (VT.SimpleTy) {
7143 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_mod_imm(RetVT, Op0, Op0IsKill, imm1);
7144 default: return 0;
7145 }
7146}
7147
7148// Top-level FastEmit function.
7149
7150unsigned fastEmit_ri_Predicate_mod_imm(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7151 switch (Opcode) {
7152 case ARMISD::CMN: return fastEmit_ARMISD_CMN_ri_Predicate_mod_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7153 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_mod_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7154 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_mod_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7155 case ARMISD::SUBS: return fastEmit_ARMISD_SUBS_ri_Predicate_mod_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7156 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_mod_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7157 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_mod_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7158 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_mod_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7159 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_mod_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7160 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_mod_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7161 default: return 0;
7162 }
7163}
7164
7165// FastEmit functions for ARMISD::SUBS.
7166
7167unsigned fastEmit_ARMISD_SUBS_MVT_i32_ri_Predicate_imm0_7(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7168 if (RetVT.SimpleTy != MVT::i32)
7169 return 0;
7170 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7171 return fastEmitInst_ri(ARM::tSUBSi3, &ARM::tGPRRegClass, Op0, Op0IsKill, imm1);
7172 }
7173 return 0;
7174}
7175
7176unsigned fastEmit_ARMISD_SUBS_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7177 switch (VT.SimpleTy) {
7178 case MVT::i32: return fastEmit_ARMISD_SUBS_MVT_i32_ri_Predicate_imm0_7(RetVT, Op0, Op0IsKill, imm1);
7179 default: return 0;
7180 }
7181}
7182
7183// FastEmit functions for ARMISD::VSHLIMM.
7184
7185unsigned fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7186 if (RetVT.SimpleTy != MVT::v16i8)
7187 return 0;
7188 if ((Subtarget->hasMVEIntegerOps())) {
7189 return fastEmitInst_ri(ARM::MVE_VSHL_immi8, &ARM::MQPRRegClass, Op0, Op0IsKill, imm1);
7190 }
7191 return 0;
7192}
7193
7194unsigned fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7195 switch (VT.SimpleTy) {
7196 case MVT::v16i8: return fastEmit_ARMISD_VSHLIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, Op0IsKill, imm1);
7197 default: return 0;
7198 }
7199}
7200
7201// FastEmit functions for ARMISD::VSHRsIMM.
7202
7203unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7204 if (RetVT.SimpleTy != MVT::v16i8)
7205 return 0;
7206 if ((Subtarget->hasMVEIntegerOps())) {
7207 return fastEmitInst_ri(ARM::MVE_VSHR_imms8, &ARM::MQPRRegClass, Op0, Op0IsKill, imm1);
7208 }
7209 return 0;
7210}
7211
7212unsigned fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7213 switch (VT.SimpleTy) {
7214 case MVT::v16i8: return fastEmit_ARMISD_VSHRsIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, Op0IsKill, imm1);
7215 default: return 0;
7216 }
7217}
7218
7219// FastEmit functions for ARMISD::VSHRuIMM.
7220
7221unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri_Predicate_imm0_7(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7222 if (RetVT.SimpleTy != MVT::v16i8)
7223 return 0;
7224 if ((Subtarget->hasMVEIntegerOps())) {
7225 return fastEmitInst_ri(ARM::MVE_VSHR_immu8, &ARM::MQPRRegClass, Op0, Op0IsKill, imm1);
7226 }
7227 return 0;
7228}
7229
7230unsigned fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7231 switch (VT.SimpleTy) {
7232 case MVT::v16i8: return fastEmit_ARMISD_VSHRuIMM_MVT_v16i8_ri_Predicate_imm0_7(RetVT, Op0, Op0IsKill, imm1);
7233 default: return 0;
7234 }
7235}
7236
7237// FastEmit functions for ISD::ADD.
7238
7239unsigned fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_7(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7240 if (RetVT.SimpleTy != MVT::i32)
7241 return 0;
7242 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7243 return fastEmitInst_ri(ARM::tADDi3, &ARM::tGPRRegClass, Op0, Op0IsKill, imm1);
7244 }
7245 return 0;
7246}
7247
7248unsigned fastEmit_ISD_ADD_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7249 switch (VT.SimpleTy) {
7250 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_7(RetVT, Op0, Op0IsKill, imm1);
7251 default: return 0;
7252 }
7253}
7254
7255// Top-level FastEmit function.
7256
7257unsigned fastEmit_ri_Predicate_imm0_7(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7258 switch (Opcode) {
7259 case ARMISD::SUBS: return fastEmit_ARMISD_SUBS_ri_Predicate_imm0_7(VT, RetVT, Op0, Op0IsKill, imm1);
7260 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, Op0IsKill, imm1);
7261 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, Op0IsKill, imm1);
7262 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_7(VT, RetVT, Op0, Op0IsKill, imm1);
7263 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_7(VT, RetVT, Op0, Op0IsKill, imm1);
7264 default: return 0;
7265 }
7266}
7267
7268// FastEmit functions for ISD::ADD.
7269
7270unsigned fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm8_255(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7271 if (RetVT.SimpleTy != MVT::i32)
7272 return 0;
7273 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7274 return fastEmitInst_ri(ARM::tADDi8, &ARM::tGPRRegClass, Op0, Op0IsKill, imm1);
7275 }
7276 return 0;
7277}
7278
7279unsigned fastEmit_ISD_ADD_ri_Predicate_imm8_255(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7280 switch (VT.SimpleTy) {
7281 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm8_255(RetVT, Op0, Op0IsKill, imm1);
7282 default: return 0;
7283 }
7284}
7285
7286// Top-level FastEmit function.
7287
7288unsigned fastEmit_ri_Predicate_imm8_255(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7289 switch (Opcode) {
7290 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm8_255(VT, RetVT, Op0, Op0IsKill, imm1);
7291 default: return 0;
7292 }
7293}
7294
7295// FastEmit functions for ARMISD::CMP.
7296
7297unsigned fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_imm0_255(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7298 if (RetVT.SimpleTy != MVT::isVoid)
7299 return 0;
7300 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7301 return fastEmitInst_ri(ARM::tCMPi8, &ARM::tGPRRegClass, Op0, Op0IsKill, imm1);
7302 }
7303 return 0;
7304}
7305
7306unsigned fastEmit_ARMISD_CMP_ri_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7307 switch (VT.SimpleTy) {
7308 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_imm0_255(RetVT, Op0, Op0IsKill, imm1);
7309 default: return 0;
7310 }
7311}
7312
7313// FastEmit functions for ARMISD::CMPZ.
7314
7315unsigned fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_imm0_255(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7316 if (RetVT.SimpleTy != MVT::isVoid)
7317 return 0;
7318 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7319 return fastEmitInst_ri(ARM::tCMPi8, &ARM::tGPRRegClass, Op0, Op0IsKill, imm1);
7320 }
7321 return 0;
7322}
7323
7324unsigned fastEmit_ARMISD_CMPZ_ri_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7325 switch (VT.SimpleTy) {
7326 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_imm0_255(RetVT, Op0, Op0IsKill, imm1);
7327 default: return 0;
7328 }
7329}
7330
7331// FastEmit functions for ARMISD::SUBS.
7332
7333unsigned fastEmit_ARMISD_SUBS_MVT_i32_ri_Predicate_imm0_255(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7334 if (RetVT.SimpleTy != MVT::i32)
7335 return 0;
7336 if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
7337 return fastEmitInst_ri(ARM::tSUBSi8, &ARM::tGPRRegClass, Op0, Op0IsKill, imm1);
7338 }
7339 return 0;
7340}
7341
7342unsigned fastEmit_ARMISD_SUBS_ri_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7343 switch (VT.SimpleTy) {
7344 case MVT::i32: return fastEmit_ARMISD_SUBS_MVT_i32_ri_Predicate_imm0_255(RetVT, Op0, Op0IsKill, imm1);
7345 default: return 0;
7346 }
7347}
7348
7349// Top-level FastEmit function.
7350
7351unsigned fastEmit_ri_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7352 switch (Opcode) {
7353 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_imm0_255(VT, RetVT, Op0, Op0IsKill, imm1);
7354 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_imm0_255(VT, RetVT, Op0, Op0IsKill, imm1);
7355 case ARMISD::SUBS: return fastEmit_ARMISD_SUBS_ri_Predicate_imm0_255(VT, RetVT, Op0, Op0IsKill, imm1);
7356 default: return 0;
7357 }
7358}
7359
7360// FastEmit functions for ARMISD::CMP.
7361
7362unsigned fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7363 if (RetVT.SimpleTy != MVT::isVoid)
7364 return 0;
7365 if ((Subtarget->isThumb2())) {
7366 return fastEmitInst_ri(ARM::t2CMPri, &ARM::GPRnopcRegClass, Op0, Op0IsKill, imm1);
7367 }
7368 return 0;
7369}
7370
7371unsigned fastEmit_ARMISD_CMP_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7372 switch (VT.SimpleTy) {
7373 case MVT::i32: return fastEmit_ARMISD_CMP_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, Op0IsKill, imm1);
7374 default: return 0;
7375 }
7376}
7377
7378// FastEmit functions for ARMISD::CMPZ.
7379
7380unsigned fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7381 if (RetVT.SimpleTy != MVT::isVoid)
7382 return 0;
7383 if ((Subtarget->isThumb2())) {
7384 return fastEmitInst_ri(ARM::t2CMPri, &ARM::GPRnopcRegClass, Op0, Op0IsKill, imm1);
7385 }
7386 return 0;
7387}
7388
7389unsigned fastEmit_ARMISD_CMPZ_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7390 switch (VT.SimpleTy) {
7391 case MVT::i32: return fastEmit_ARMISD_CMPZ_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, Op0IsKill, imm1);
7392 default: return 0;
7393 }
7394}
7395
7396// FastEmit functions for ARMISD::SUBS.
7397
7398unsigned fastEmit_ARMISD_SUBS_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7399 if (RetVT.SimpleTy != MVT::i32)
7400 return 0;
7401 if ((Subtarget->isThumb2())) {
7402 return fastEmitInst_ri(ARM::t2SUBSri, &ARM::rGPRRegClass, Op0, Op0IsKill, imm1);
7403 }
7404 return 0;
7405}
7406
7407unsigned fastEmit_ARMISD_SUBS_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7408 switch (VT.SimpleTy) {
7409 case MVT::i32: return fastEmit_ARMISD_SUBS_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, Op0IsKill, imm1);
7410 default: return 0;
7411 }
7412}
7413
7414// FastEmit functions for ISD::ADD.
7415
7416unsigned fastEmit_ISD_ADD_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7417 if (RetVT.SimpleTy != MVT::i32)
7418 return 0;
7419 if ((Subtarget->isThumb2())) {
7420 return fastEmitInst_ri(ARM::t2ADDri, &ARM::rGPRRegClass, Op0, Op0IsKill, imm1);
7421 }
7422 return 0;
7423}
7424
7425unsigned fastEmit_ISD_ADD_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7426 switch (VT.SimpleTy) {
7427 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, Op0IsKill, imm1);
7428 default: return 0;
7429 }
7430}
7431
7432// FastEmit functions for ISD::AND.
7433
7434unsigned fastEmit_ISD_AND_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7435 if (RetVT.SimpleTy != MVT::i32)
7436 return 0;
7437 if ((Subtarget->isThumb2())) {
7438 return fastEmitInst_ri(ARM::t2ANDri, &ARM::rGPRRegClass, Op0, Op0IsKill, imm1);
7439 }
7440 return 0;
7441}
7442
7443unsigned fastEmit_ISD_AND_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7444 switch (VT.SimpleTy) {
7445 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, Op0IsKill, imm1);
7446 default: return 0;
7447 }
7448}
7449
7450// FastEmit functions for ISD::OR.
7451
7452unsigned fastEmit_ISD_OR_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7453 if (RetVT.SimpleTy != MVT::i32)
7454 return 0;
7455 if ((Subtarget->isThumb2())) {
7456 return fastEmitInst_ri(ARM::t2ORRri, &ARM::rGPRRegClass, Op0, Op0IsKill, imm1);
7457 }
7458 return 0;
7459}
7460
7461unsigned fastEmit_ISD_OR_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7462 switch (VT.SimpleTy) {
7463 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, Op0IsKill, imm1);
7464 default: return 0;
7465 }
7466}
7467
7468// FastEmit functions for ISD::SUB.
7469
7470unsigned fastEmit_ISD_SUB_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7471 if (RetVT.SimpleTy != MVT::i32)
7472 return 0;
7473 if ((Subtarget->isThumb2())) {
7474 return fastEmitInst_ri(ARM::t2SUBri, &ARM::rGPRRegClass, Op0, Op0IsKill, imm1);
7475 }
7476 return 0;
7477}
7478
7479unsigned fastEmit_ISD_SUB_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7480 switch (VT.SimpleTy) {
7481 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, Op0IsKill, imm1);
7482 default: return 0;
7483 }
7484}
7485
7486// FastEmit functions for ISD::XOR.
7487
7488unsigned fastEmit_ISD_XOR_MVT_i32_ri_Predicate_t2_so_imm(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7489 if (RetVT.SimpleTy != MVT::i32)
7490 return 0;
7491 if ((Subtarget->isThumb2())) {
7492 return fastEmitInst_ri(ARM::t2EORri, &ARM::rGPRRegClass, Op0, Op0IsKill, imm1);
7493 }
7494 return 0;
7495}
7496
7497unsigned fastEmit_ISD_XOR_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7498 switch (VT.SimpleTy) {
7499 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_t2_so_imm(RetVT, Op0, Op0IsKill, imm1);
7500 default: return 0;
7501 }
7502}
7503
7504// Top-level FastEmit function.
7505
7506unsigned fastEmit_ri_Predicate_t2_so_imm(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7507 switch (Opcode) {
7508 case ARMISD::CMP: return fastEmit_ARMISD_CMP_ri_Predicate_t2_so_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7509 case ARMISD::CMPZ: return fastEmit_ARMISD_CMPZ_ri_Predicate_t2_so_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7510 case ARMISD::SUBS: return fastEmit_ARMISD_SUBS_ri_Predicate_t2_so_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7511 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_t2_so_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7512 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_t2_so_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7513 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_t2_so_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7514 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_t2_so_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7515 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_t2_so_imm(VT, RetVT, Op0, Op0IsKill, imm1);
7516 default: return 0;
7517 }
7518}
7519
7520// FastEmit functions for ISD::ADD.
7521
7522unsigned fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_4095(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7523 if (RetVT.SimpleTy != MVT::i32)
7524 return 0;
7525 if ((Subtarget->isThumb2())) {
7526 return fastEmitInst_ri(ARM::t2ADDri12, &ARM::rGPRRegClass, Op0, Op0IsKill, imm1);
7527 }
7528 return 0;
7529}
7530
7531unsigned fastEmit_ISD_ADD_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7532 switch (VT.SimpleTy) {
7533 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_imm0_4095(RetVT, Op0, Op0IsKill, imm1);
7534 default: return 0;
7535 }
7536}
7537
7538// FastEmit functions for ISD::SUB.
7539
7540unsigned fastEmit_ISD_SUB_MVT_i32_ri_Predicate_imm0_4095(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7541 if (RetVT.SimpleTy != MVT::i32)
7542 return 0;
7543 if ((Subtarget->isThumb2())) {
7544 return fastEmitInst_ri(ARM::t2SUBri12, &ARM::rGPRRegClass, Op0, Op0IsKill, imm1);
7545 }
7546 return 0;
7547}
7548
7549unsigned fastEmit_ISD_SUB_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7550 switch (VT.SimpleTy) {
7551 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_imm0_4095(RetVT, Op0, Op0IsKill, imm1);
7552 default: return 0;
7553 }
7554}
7555
7556// Top-level FastEmit function.
7557
7558unsigned fastEmit_ri_Predicate_imm0_4095(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7559 switch (Opcode) {
7560 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_imm0_4095(VT, RetVT, Op0, Op0IsKill, imm1);
7561 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_imm0_4095(VT, RetVT, Op0, Op0IsKill, imm1);
7562 default: return 0;
7563 }
7564}
7565
7566// FastEmit functions for ISD::SHL.
7567
7568unsigned fastEmit_ISD_SHL_MVT_i32_ri_Predicate_imm1_31(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7569 if (RetVT.SimpleTy != MVT::i32)
7570 return 0;
7571 if ((Subtarget->isThumb2())) {
7572 return fastEmitInst_ri(ARM::t2LSLri, &ARM::rGPRRegClass, Op0, Op0IsKill, imm1);
7573 }
7574 return 0;
7575}
7576
7577unsigned fastEmit_ISD_SHL_ri_Predicate_imm1_31(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7578 switch (VT.SimpleTy) {
7579 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri_Predicate_imm1_31(RetVT, Op0, Op0IsKill, imm1);
7580 default: return 0;
7581 }
7582}
7583
7584// Top-level FastEmit function.
7585
7586unsigned fastEmit_ri_Predicate_imm1_31(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7587 switch (Opcode) {
7588 case ISD::SHL: return fastEmit_ISD_SHL_ri_Predicate_imm1_31(VT, RetVT, Op0, Op0IsKill, imm1);
7589 default: return 0;
7590 }
7591}
7592
7593// FastEmit functions for ARMISD::VSHLIMM.
7594
7595unsigned fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7596 if (RetVT.SimpleTy != MVT::v4i32)
7597 return 0;
7598 if ((Subtarget->hasMVEIntegerOps())) {
7599 return fastEmitInst_ri(ARM::MVE_VSHL_immi32, &ARM::MQPRRegClass, Op0, Op0IsKill, imm1);
7600 }
7601 return 0;
7602}
7603
7604unsigned fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7605 switch (VT.SimpleTy) {
7606 case MVT::v4i32: return fastEmit_ARMISD_VSHLIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, Op0IsKill, imm1);
7607 default: return 0;
7608 }
7609}
7610
7611// FastEmit functions for ARMISD::VSHRsIMM.
7612
7613unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7614 if (RetVT.SimpleTy != MVT::v4i32)
7615 return 0;
7616 if ((Subtarget->hasMVEIntegerOps())) {
7617 return fastEmitInst_ri(ARM::MVE_VSHR_imms32, &ARM::MQPRRegClass, Op0, Op0IsKill, imm1);
7618 }
7619 return 0;
7620}
7621
7622unsigned fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7623 switch (VT.SimpleTy) {
7624 case MVT::v4i32: return fastEmit_ARMISD_VSHRsIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, Op0IsKill, imm1);
7625 default: return 0;
7626 }
7627}
7628
7629// FastEmit functions for ARMISD::VSHRuIMM.
7630
7631unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri_Predicate_imm0_31(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7632 if (RetVT.SimpleTy != MVT::v4i32)
7633 return 0;
7634 if ((Subtarget->hasMVEIntegerOps())) {
7635 return fastEmitInst_ri(ARM::MVE_VSHR_immu32, &ARM::MQPRRegClass, Op0, Op0IsKill, imm1);
7636 }
7637 return 0;
7638}
7639
7640unsigned fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_31(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7641 switch (VT.SimpleTy) {
7642 case MVT::v4i32: return fastEmit_ARMISD_VSHRuIMM_MVT_v4i32_ri_Predicate_imm0_31(RetVT, Op0, Op0IsKill, imm1);
7643 default: return 0;
7644 }
7645}
7646
7647// FastEmit functions for ISD::ROTR.
7648
7649unsigned fastEmit_ISD_ROTR_MVT_i32_ri_Predicate_imm0_31(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7650 if (RetVT.SimpleTy != MVT::i32)
7651 return 0;
7652 if ((Subtarget->isThumb2())) {
7653 return fastEmitInst_ri(ARM::t2RORri, &ARM::rGPRRegClass, Op0, Op0IsKill, imm1);
7654 }
7655 return 0;
7656}
7657
7658unsigned fastEmit_ISD_ROTR_ri_Predicate_imm0_31(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7659 switch (VT.SimpleTy) {
7660 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri_Predicate_imm0_31(RetVT, Op0, Op0IsKill, imm1);
7661 default: return 0;
7662 }
7663}
7664
7665// Top-level FastEmit function.
7666
7667unsigned fastEmit_ri_Predicate_imm0_31(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7668 switch (Opcode) {
7669 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, Op0IsKill, imm1);
7670 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, Op0IsKill, imm1);
7671 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_31(VT, RetVT, Op0, Op0IsKill, imm1);
7672 case ISD::ROTR: return fastEmit_ISD_ROTR_ri_Predicate_imm0_31(VT, RetVT, Op0, Op0IsKill, imm1);
7673 default: return 0;
7674 }
7675}
7676
7677// FastEmit functions for ARMISD::VQRSHRNsIMM.
7678
7679unsigned fastEmit_ARMISD_VQRSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7680 if (RetVT.SimpleTy != MVT::v8i8)
7681 return 0;
7682 if ((Subtarget->hasNEON())) {
7683 return fastEmitInst_ri(ARM::VQRSHRNsv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
7684 }
7685 return 0;
7686}
7687
7688unsigned fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7689 switch (VT.SimpleTy) {
7690 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, Op0IsKill, imm1);
7691 default: return 0;
7692 }
7693}
7694
7695// FastEmit functions for ARMISD::VQRSHRNsuIMM.
7696
7697unsigned fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7698 if (RetVT.SimpleTy != MVT::v8i8)
7699 return 0;
7700 if ((Subtarget->hasNEON())) {
7701 return fastEmitInst_ri(ARM::VQRSHRUNv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
7702 }
7703 return 0;
7704}
7705
7706unsigned fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7707 switch (VT.SimpleTy) {
7708 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, Op0IsKill, imm1);
7709 default: return 0;
7710 }
7711}
7712
7713// FastEmit functions for ARMISD::VQRSHRNuIMM.
7714
7715unsigned fastEmit_ARMISD_VQRSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7716 if (RetVT.SimpleTy != MVT::v8i8)
7717 return 0;
7718 if ((Subtarget->hasNEON())) {
7719 return fastEmitInst_ri(ARM::VQRSHRNuv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
7720 }
7721 return 0;
7722}
7723
7724unsigned fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7725 switch (VT.SimpleTy) {
7726 case MVT::v8i16: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, Op0IsKill, imm1);
7727 default: return 0;
7728 }
7729}
7730
7731// FastEmit functions for ARMISD::VQSHRNsIMM.
7732
7733unsigned fastEmit_ARMISD_VQSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7734 if (RetVT.SimpleTy != MVT::v8i8)
7735 return 0;
7736 if ((Subtarget->hasNEON())) {
7737 return fastEmitInst_ri(ARM::VQSHRNsv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
7738 }
7739 return 0;
7740}
7741
7742unsigned fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7743 switch (VT.SimpleTy) {
7744 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, Op0IsKill, imm1);
7745 default: return 0;
7746 }
7747}
7748
7749// FastEmit functions for ARMISD::VQSHRNsuIMM.
7750
7751unsigned fastEmit_ARMISD_VQSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7752 if (RetVT.SimpleTy != MVT::v8i8)
7753 return 0;
7754 if ((Subtarget->hasNEON())) {
7755 return fastEmitInst_ri(ARM::VQSHRUNv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
7756 }
7757 return 0;
7758}
7759
7760unsigned fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7761 switch (VT.SimpleTy) {
7762 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, Op0IsKill, imm1);
7763 default: return 0;
7764 }
7765}
7766
7767// FastEmit functions for ARMISD::VQSHRNuIMM.
7768
7769unsigned fastEmit_ARMISD_VQSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7770 if (RetVT.SimpleTy != MVT::v8i8)
7771 return 0;
7772 if ((Subtarget->hasNEON())) {
7773 return fastEmitInst_ri(ARM::VQSHRNuv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
7774 }
7775 return 0;
7776}
7777
7778unsigned fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7779 switch (VT.SimpleTy) {
7780 case MVT::v8i16: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, Op0IsKill, imm1);
7781 default: return 0;
7782 }
7783}
7784
7785// FastEmit functions for ARMISD::VRSHRNIMM.
7786
7787unsigned fastEmit_ARMISD_VRSHRNIMM_MVT_v8i16_ri_Predicate_shr_imm8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7788 if (RetVT.SimpleTy != MVT::v8i8)
7789 return 0;
7790 if ((Subtarget->hasNEON())) {
7791 return fastEmitInst_ri(ARM::VRSHRNv8i8, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
7792 }
7793 return 0;
7794}
7795
7796unsigned fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7797 switch (VT.SimpleTy) {
7798 case MVT::v8i16: return fastEmit_ARMISD_VRSHRNIMM_MVT_v8i16_ri_Predicate_shr_imm8(RetVT, Op0, Op0IsKill, imm1);
7799 default: return 0;
7800 }
7801}
7802
7803// Top-level FastEmit function.
7804
7805unsigned fastEmit_ri_Predicate_shr_imm8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7806 switch (Opcode) {
7807 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, Op0IsKill, imm1);
7808 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, Op0IsKill, imm1);
7809 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, Op0IsKill, imm1);
7810 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, Op0IsKill, imm1);
7811 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, Op0IsKill, imm1);
7812 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, Op0IsKill, imm1);
7813 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm8(VT, RetVT, Op0, Op0IsKill, imm1);
7814 default: return 0;
7815 }
7816}
7817
7818// FastEmit functions for ARMISD::VQRSHRNsIMM.
7819
7820unsigned fastEmit_ARMISD_VQRSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7821 if (RetVT.SimpleTy != MVT::v4i16)
7822 return 0;
7823 if ((Subtarget->hasNEON())) {
7824 return fastEmitInst_ri(ARM::VQRSHRNsv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
7825 }
7826 return 0;
7827}
7828
7829unsigned fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7830 switch (VT.SimpleTy) {
7831 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, Op0IsKill, imm1);
7832 default: return 0;
7833 }
7834}
7835
7836// FastEmit functions for ARMISD::VQRSHRNsuIMM.
7837
7838unsigned fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7839 if (RetVT.SimpleTy != MVT::v4i16)
7840 return 0;
7841 if ((Subtarget->hasNEON())) {
7842 return fastEmitInst_ri(ARM::VQRSHRUNv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
7843 }
7844 return 0;
7845}
7846
7847unsigned fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7848 switch (VT.SimpleTy) {
7849 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, Op0IsKill, imm1);
7850 default: return 0;
7851 }
7852}
7853
7854// FastEmit functions for ARMISD::VQRSHRNuIMM.
7855
7856unsigned fastEmit_ARMISD_VQRSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7857 if (RetVT.SimpleTy != MVT::v4i16)
7858 return 0;
7859 if ((Subtarget->hasNEON())) {
7860 return fastEmitInst_ri(ARM::VQRSHRNuv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
7861 }
7862 return 0;
7863}
7864
7865unsigned fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7866 switch (VT.SimpleTy) {
7867 case MVT::v4i32: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, Op0IsKill, imm1);
7868 default: return 0;
7869 }
7870}
7871
7872// FastEmit functions for ARMISD::VQSHRNsIMM.
7873
7874unsigned fastEmit_ARMISD_VQSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7875 if (RetVT.SimpleTy != MVT::v4i16)
7876 return 0;
7877 if ((Subtarget->hasNEON())) {
7878 return fastEmitInst_ri(ARM::VQSHRNsv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
7879 }
7880 return 0;
7881}
7882
7883unsigned fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7884 switch (VT.SimpleTy) {
7885 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, Op0IsKill, imm1);
7886 default: return 0;
7887 }
7888}
7889
7890// FastEmit functions for ARMISD::VQSHRNsuIMM.
7891
7892unsigned fastEmit_ARMISD_VQSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7893 if (RetVT.SimpleTy != MVT::v4i16)
7894 return 0;
7895 if ((Subtarget->hasNEON())) {
7896 return fastEmitInst_ri(ARM::VQSHRUNv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
7897 }
7898 return 0;
7899}
7900
7901unsigned fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7902 switch (VT.SimpleTy) {
7903 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, Op0IsKill, imm1);
7904 default: return 0;
7905 }
7906}
7907
7908// FastEmit functions for ARMISD::VQSHRNuIMM.
7909
7910unsigned fastEmit_ARMISD_VQSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7911 if (RetVT.SimpleTy != MVT::v4i16)
7912 return 0;
7913 if ((Subtarget->hasNEON())) {
7914 return fastEmitInst_ri(ARM::VQSHRNuv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
7915 }
7916 return 0;
7917}
7918
7919unsigned fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7920 switch (VT.SimpleTy) {
7921 case MVT::v4i32: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, Op0IsKill, imm1);
7922 default: return 0;
7923 }
7924}
7925
7926// FastEmit functions for ARMISD::VRSHRNIMM.
7927
7928unsigned fastEmit_ARMISD_VRSHRNIMM_MVT_v4i32_ri_Predicate_shr_imm16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7929 if (RetVT.SimpleTy != MVT::v4i16)
7930 return 0;
7931 if ((Subtarget->hasNEON())) {
7932 return fastEmitInst_ri(ARM::VRSHRNv4i16, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
7933 }
7934 return 0;
7935}
7936
7937unsigned fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7938 switch (VT.SimpleTy) {
7939 case MVT::v4i32: return fastEmit_ARMISD_VRSHRNIMM_MVT_v4i32_ri_Predicate_shr_imm16(RetVT, Op0, Op0IsKill, imm1);
7940 default: return 0;
7941 }
7942}
7943
7944// Top-level FastEmit function.
7945
7946unsigned fastEmit_ri_Predicate_shr_imm16(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7947 switch (Opcode) {
7948 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, Op0IsKill, imm1);
7949 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, Op0IsKill, imm1);
7950 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, Op0IsKill, imm1);
7951 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, Op0IsKill, imm1);
7952 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, Op0IsKill, imm1);
7953 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, Op0IsKill, imm1);
7954 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm16(VT, RetVT, Op0, Op0IsKill, imm1);
7955 default: return 0;
7956 }
7957}
7958
7959// FastEmit functions for ARMISD::VQRSHRNsIMM.
7960
7961unsigned fastEmit_ARMISD_VQRSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7962 if (RetVT.SimpleTy != MVT::v2i32)
7963 return 0;
7964 if ((Subtarget->hasNEON())) {
7965 return fastEmitInst_ri(ARM::VQRSHRNsv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
7966 }
7967 return 0;
7968}
7969
7970unsigned fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7971 switch (VT.SimpleTy) {
7972 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, Op0IsKill, imm1);
7973 default: return 0;
7974 }
7975}
7976
7977// FastEmit functions for ARMISD::VQRSHRNsuIMM.
7978
7979unsigned fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7980 if (RetVT.SimpleTy != MVT::v2i32)
7981 return 0;
7982 if ((Subtarget->hasNEON())) {
7983 return fastEmitInst_ri(ARM::VQRSHRUNv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
7984 }
7985 return 0;
7986}
7987
7988unsigned fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7989 switch (VT.SimpleTy) {
7990 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, Op0IsKill, imm1);
7991 default: return 0;
7992 }
7993}
7994
7995// FastEmit functions for ARMISD::VQRSHRNuIMM.
7996
7997unsigned fastEmit_ARMISD_VQRSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
7998 if (RetVT.SimpleTy != MVT::v2i32)
7999 return 0;
8000 if ((Subtarget->hasNEON())) {
8001 return fastEmitInst_ri(ARM::VQRSHRNuv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
8002 }
8003 return 0;
8004}
8005
8006unsigned fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8007 switch (VT.SimpleTy) {
8008 case MVT::v2i64: return fastEmit_ARMISD_VQRSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, Op0IsKill, imm1);
8009 default: return 0;
8010 }
8011}
8012
8013// FastEmit functions for ARMISD::VQSHRNsIMM.
8014
8015unsigned fastEmit_ARMISD_VQSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8016 if (RetVT.SimpleTy != MVT::v2i32)
8017 return 0;
8018 if ((Subtarget->hasNEON())) {
8019 return fastEmitInst_ri(ARM::VQSHRNsv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
8020 }
8021 return 0;
8022}
8023
8024unsigned fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8025 switch (VT.SimpleTy) {
8026 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNsIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, Op0IsKill, imm1);
8027 default: return 0;
8028 }
8029}
8030
8031// FastEmit functions for ARMISD::VQSHRNsuIMM.
8032
8033unsigned fastEmit_ARMISD_VQSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8034 if (RetVT.SimpleTy != MVT::v2i32)
8035 return 0;
8036 if ((Subtarget->hasNEON())) {
8037 return fastEmitInst_ri(ARM::VQSHRUNv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
8038 }
8039 return 0;
8040}
8041
8042unsigned fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8043 switch (VT.SimpleTy) {
8044 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNsuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, Op0IsKill, imm1);
8045 default: return 0;
8046 }
8047}
8048
8049// FastEmit functions for ARMISD::VQSHRNuIMM.
8050
8051unsigned fastEmit_ARMISD_VQSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8052 if (RetVT.SimpleTy != MVT::v2i32)
8053 return 0;
8054 if ((Subtarget->hasNEON())) {
8055 return fastEmitInst_ri(ARM::VQSHRNuv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
8056 }
8057 return 0;
8058}
8059
8060unsigned fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8061 switch (VT.SimpleTy) {
8062 case MVT::v2i64: return fastEmit_ARMISD_VQSHRNuIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, Op0IsKill, imm1);
8063 default: return 0;
8064 }
8065}
8066
8067// FastEmit functions for ARMISD::VRSHRNIMM.
8068
8069unsigned fastEmit_ARMISD_VRSHRNIMM_MVT_v2i64_ri_Predicate_shr_imm32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8070 if (RetVT.SimpleTy != MVT::v2i32)
8071 return 0;
8072 if ((Subtarget->hasNEON())) {
8073 return fastEmitInst_ri(ARM::VRSHRNv2i32, &ARM::DPRRegClass, Op0, Op0IsKill, imm1);
8074 }
8075 return 0;
8076}
8077
8078unsigned fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8079 switch (VT.SimpleTy) {
8080 case MVT::v2i64: return fastEmit_ARMISD_VRSHRNIMM_MVT_v2i64_ri_Predicate_shr_imm32(RetVT, Op0, Op0IsKill, imm1);
8081 default: return 0;
8082 }
8083}
8084
8085// Top-level FastEmit function.
8086
8087unsigned fastEmit_ri_Predicate_shr_imm32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8088 switch (Opcode) {
8089 case ARMISD::VQRSHRNsIMM: return fastEmit_ARMISD_VQRSHRNsIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, Op0IsKill, imm1);
8090 case ARMISD::VQRSHRNsuIMM: return fastEmit_ARMISD_VQRSHRNsuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, Op0IsKill, imm1);
8091 case ARMISD::VQRSHRNuIMM: return fastEmit_ARMISD_VQRSHRNuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, Op0IsKill, imm1);
8092 case ARMISD::VQSHRNsIMM: return fastEmit_ARMISD_VQSHRNsIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, Op0IsKill, imm1);
8093 case ARMISD::VQSHRNsuIMM: return fastEmit_ARMISD_VQSHRNsuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, Op0IsKill, imm1);
8094 case ARMISD::VQSHRNuIMM: return fastEmit_ARMISD_VQSHRNuIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, Op0IsKill, imm1);
8095 case ARMISD::VRSHRNIMM: return fastEmit_ARMISD_VRSHRNIMM_ri_Predicate_shr_imm32(VT, RetVT, Op0, Op0IsKill, imm1);
8096 default: return 0;
8097 }
8098}
8099
8100// FastEmit functions for ARMISD::VDUPLANE.
8101
8102unsigned fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri_Predicate_VectorIndex32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8103 if (RetVT.SimpleTy != MVT::v16i8)
8104 return 0;
8105 if ((Subtarget->hasNEON())) {
8106 return fastEmitInst_ri(ARM::VDUPLN8q, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
8107 }
8108 return 0;
8109}
8110
8111unsigned fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri_Predicate_VectorIndex32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8112 if (RetVT.SimpleTy != MVT::v8i16)
8113 return 0;
8114 if ((Subtarget->hasNEON())) {
8115 return fastEmitInst_ri(ARM::VDUPLN16q, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
8116 }
8117 return 0;
8118}
8119
8120unsigned fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri_Predicate_VectorIndex32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8121 if (RetVT.SimpleTy != MVT::v4i32)
8122 return 0;
8123 if ((Subtarget->hasNEON())) {
8124 return fastEmitInst_ri(ARM::VDUPLN32q, &ARM::QPRRegClass, Op0, Op0IsKill, imm1);
8125 }
8126 return 0;
8127}
8128
8129unsigned fastEmit_ARMISD_VDUPLANE_ri_Predicate_VectorIndex32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8130 switch (VT.SimpleTy) {
8131 case MVT::v8i8: return fastEmit_ARMISD_VDUPLANE_MVT_v8i8_ri_Predicate_VectorIndex32(RetVT, Op0, Op0IsKill, imm1);
8132 case MVT::v4i16: return fastEmit_ARMISD_VDUPLANE_MVT_v4i16_ri_Predicate_VectorIndex32(RetVT, Op0, Op0IsKill, imm1);
8133 case MVT::v2i32: return fastEmit_ARMISD_VDUPLANE_MVT_v2i32_ri_Predicate_VectorIndex32(RetVT, Op0, Op0IsKill, imm1);
8134 default: return 0;
8135 }
8136}
8137
8138// Top-level FastEmit function.
8139
8140unsigned fastEmit_ri_Predicate_VectorIndex32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8141 switch (Opcode) {
8142 case ARMISD::VDUPLANE: return fastEmit_ARMISD_VDUPLANE_ri_Predicate_VectorIndex32(VT, RetVT, Op0, Op0IsKill, imm1);
8143 default: return 0;
8144 }
8145}
8146
8147// FastEmit functions for ARMISD::VSHLIMM.
8148
8149unsigned fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8150 if (RetVT.SimpleTy != MVT::v8i16)
8151 return 0;
8152 if ((Subtarget->hasMVEIntegerOps())) {
8153 return fastEmitInst_ri(ARM::MVE_VSHL_immi16, &ARM::MQPRRegClass, Op0, Op0IsKill, imm1);
8154 }
8155 return 0;
8156}
8157
8158unsigned fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8159 switch (VT.SimpleTy) {
8160 case MVT::v8i16: return fastEmit_ARMISD_VSHLIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, Op0IsKill, imm1);
8161 default: return 0;
8162 }
8163}
8164
8165// FastEmit functions for ARMISD::VSHRsIMM.
8166
8167unsigned fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8168 if (RetVT.SimpleTy != MVT::v8i16)
8169 return 0;
8170 if ((Subtarget->hasMVEIntegerOps())) {
8171 return fastEmitInst_ri(ARM::MVE_VSHR_imms16, &ARM::MQPRRegClass, Op0, Op0IsKill, imm1);
8172 }
8173 return 0;
8174}
8175
8176unsigned fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8177 switch (VT.SimpleTy) {
8178 case MVT::v8i16: return fastEmit_ARMISD_VSHRsIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, Op0IsKill, imm1);
8179 default: return 0;
8180 }
8181}
8182
8183// FastEmit functions for ARMISD::VSHRuIMM.
8184
8185unsigned fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri_Predicate_imm0_15(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8186 if (RetVT.SimpleTy != MVT::v8i16)
8187 return 0;
8188 if ((Subtarget->hasMVEIntegerOps())) {
8189 return fastEmitInst_ri(ARM::MVE_VSHR_immu16, &ARM::MQPRRegClass, Op0, Op0IsKill, imm1);
8190 }
8191 return 0;
8192}
8193
8194unsigned fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_15(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8195 switch (VT.SimpleTy) {
8196 case MVT::v8i16: return fastEmit_ARMISD_VSHRuIMM_MVT_v8i16_ri_Predicate_imm0_15(RetVT, Op0, Op0IsKill, imm1);
8197 default: return 0;
8198 }
8199}
8200
8201// Top-level FastEmit function.
8202
8203unsigned fastEmit_ri_Predicate_imm0_15(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
8204 switch (Opcode) {
8205 case ARMISD::VSHLIMM: return fastEmit_ARMISD_VSHLIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, Op0IsKill, imm1);
8206 case ARMISD::VSHRsIMM: return fastEmit_ARMISD_VSHRsIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, Op0IsKill, imm1);
8207 case ARMISD::VSHRuIMM: return fastEmit_ARMISD_VSHRuIMM_ri_Predicate_imm0_15(VT, RetVT, Op0, Op0IsKill, imm1);
8208 default: return 0;
8209 }
8210}
8211
8212// FastEmit functions for ISD::Constant.
8213
8214unsigned fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
8215 if (RetVT.SimpleTy != MVT::i32)
8216 return 0;
8217 if ((Subtarget->isThumb()) && (Subtarget->useMovt())) {
8218 return fastEmitInst_i(ARM::t2MOVi32imm, &ARM::rGPRRegClass, imm0);
8219 }
8220 return 0;
8221}
8222
8223unsigned fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
8224 switch (VT.SimpleTy) {
8225 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
8226 default: return 0;
8227 }
8228}
8229
8230// Top-level FastEmit function.
8231
8232unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
8233 switch (Opcode) {
8234 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
8235 default: return 0;
8236 }
8237}
8238
8239